gcc/config/ia64/ia64.c

   1 /* Definitions of target machine for GNU compiler.
   2    Copyright (C) 1999-2019 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson <wilson@cygnus.com> and
   4                   David Mosberger <davidm@hpl.hp.com>.
   5
   6 This file is part of GCC.
   7
   8 GCC is free software; you can redistribute it and/or modify
   9 it under the terms of the GNU General Public License as published by
  10 the Free Software Foundation; either version 3, or (at your option)
  11 any later version.
  12
  13 GCC is distributed in the hope that it will be useful,
  14 but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 GNU General Public License for more details.
  17
  18 You should have received a copy of the GNU General Public License
  19 along with GCC; see the file COPYING3.  If not see
  20 <http://www.gnu.org/licenses/>.  */
  21
  22 #define IN_TARGET_CODE 1
  23
  24 #include "config.h"
  25 #include "system.h"
  26 #include "coretypes.h"
  27 #include "backend.h"
  28 #include "target.h"
  29 #include "rtl.h"
  30 #include "tree.h"
  31 #include "memmodel.h"
  32 #include "cfghooks.h"
  33 #include "df.h"
  34 #include "tm_p.h"
  35 #include "stringpool.h"
  36 #include "attribs.h"
  37 #include "optabs.h"
  38 #include "regs.h"
  39 #include "emit-rtl.h"
  40 #include "recog.h"
  41 #include "diagnostic-core.h"
  42 #include "alias.h"
  43 #include "fold-const.h"
  44 #include "stor-layout.h"
  45 #include "calls.h"
  46 #include "varasm.h"
  47 #include "output.h"
  48 #include "insn-attr.h"
  49 #include "flags.h"
  50 #include "explow.h"
  51 #include "expr.h"
  52 #include "cfgrtl.h"
  53 #include "libfuncs.h"
  54 #include "sched-int.h"
  55 #include "common/common-target.h"
  56 #include "langhooks.h"
  57 #include "gimplify.h"
  58 #include "intl.h"
  59 #include "debug.h"
  60 #include "params.h"
  61 #include "dbgcnt.h"
  62 #include "tm-constrs.h"
  63 #include "sel-sched.h"
  64 #include "reload.h"
  65 #include "opts.h"
  66 #include "dumpfile.h"
  67 #include "builtins.h"
  68
  69 /* This file should be included last.  */
  70 #include "target-def.h"
  71
  72 /* This is used for communication between ASM_OUTPUT_LABEL and
  73    ASM_OUTPUT_LABELREF.  */
  74 int ia64_asm_output_label = 0;
  75
  76 /* Register names for ia64_expand_prologue.  */
  77 static const char * const ia64_reg_numbers[96] =
  78 { "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
  79   "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
  80   "r48", "r49", "r50", "r51", "r52", "r53", "r54", "r55",
  81   "r56", "r57", "r58", "r59", "r60", "r61", "r62", "r63",
  82   "r64", "r65", "r66", "r67", "r68", "r69", "r70", "r71",
  83   "r72", "r73", "r74", "r75", "r76", "r77", "r78", "r79",
  84   "r80", "r81", "r82", "r83", "r84", "r85", "r86", "r87",
  85   "r88", "r89", "r90", "r91", "r92", "r93", "r94", "r95",
  86   "r96", "r97", "r98", "r99", "r100","r101","r102","r103",
  87   "r104","r105","r106","r107","r108","r109","r110","r111",
  88   "r112","r113","r114","r115","r116","r117","r118","r119",
  89   "r120","r121","r122","r123","r124","r125","r126","r127"};
  90
  91 /* ??? These strings could be shared with REGISTER_NAMES.  */
  92 static const char * const ia64_input_reg_names[8] =
  93 { "in0",  "in1",  "in2",  "in3",  "in4",  "in5",  "in6",  "in7" };
  94
  95 /* ??? These strings could be shared with REGISTER_NAMES.  */
  96 static const char * const ia64_local_reg_names[80] =
  97 { "loc0", "loc1", "loc2", "loc3", "loc4", "loc5", "loc6", "loc7",
  98   "loc8", "loc9", "loc10","loc11","loc12","loc13","loc14","loc15",
  99   "loc16","loc17","loc18","loc19","loc20","loc21","loc22","loc23",
 100   "loc24","loc25","loc26","loc27","loc28","loc29","loc30","loc31",
 101   "loc32","loc33","loc34","loc35","loc36","loc37","loc38","loc39",
 102   "loc40","loc41","loc42","loc43","loc44","loc45","loc46","loc47",
 103   "loc48","loc49","loc50","loc51","loc52","loc53","loc54","loc55",
 104   "loc56","loc57","loc58","loc59","loc60","loc61","loc62","loc63",
 105   "loc64","loc65","loc66","loc67","loc68","loc69","loc70","loc71",
 106   "loc72","loc73","loc74","loc75","loc76","loc77","loc78","loc79" };
 107
 108 /* ??? These strings could be shared with REGISTER_NAMES.  */
 109 static const char * const ia64_output_reg_names[8] =
 110 { "out0", "out1", "out2", "out3", "out4", "out5", "out6", "out7" };
 111
 112 /* Variables which are this size or smaller are put in the sdata/sbss
 113    sections.  */
 114
 115 unsigned int ia64_section_threshold;
 116
 117 /* The following variable is used by the DFA insn scheduler.  The value is
 118    TRUE if we do insn bundling instead of insn scheduling.  */
 119 int bundling_p = 0;
 120
 121 enum ia64_frame_regs
 122 {
 123    reg_fp,
 124    reg_save_b0,
 125    reg_save_pr,
 126    reg_save_ar_pfs,
 127    reg_save_ar_unat,
 128    reg_save_ar_lc,
 129    reg_save_gp,
 130    number_of_ia64_frame_regs
 131 };
 132
 133 /* Structure to be filled in by ia64_compute_frame_size with register
 134    save masks and offsets for the current function.  */
 135
 136 struct ia64_frame_info
 137 {
 138   HOST_WIDE_INT total_size;     /* size of the stack frame, not including
 139                                    the caller's scratch area.  */
 140   HOST_WIDE_INT spill_cfa_off;  /* top of the reg spill area from the cfa.  */
 141   HOST_WIDE_INT spill_size;     /* size of the gr/br/fr spill area.  */
 142   HOST_WIDE_INT extra_spill_size;  /* size of spill area for others.  */
 143   HARD_REG_SET mask;            /* mask of saved registers.  */
 144   unsigned int gr_used_mask;    /* mask of registers in use as gr spill
 145                                    registers or long-term scratches.  */
 146   int n_spilled;                /* number of spilled registers.  */
 147   int r[number_of_ia64_frame_regs];  /* Frame related registers.  */
 148   int n_input_regs;             /* number of input registers used.  */
 149   int n_local_regs;             /* number of local registers used.  */
 150   int n_output_regs;            /* number of output registers used.  */
 151   int n_rotate_regs;            /* number of rotating registers used.  */
 152
 153   char need_regstk;             /* true if a .regstk directive needed.  */
 154   char initialized;             /* true if the data is finalized.  */
 155 };
 156
 157 /* Current frame information calculated by ia64_compute_frame_size.  */
 158 static struct ia64_frame_info current_frame_info;
 159 /* The actual registers that are emitted.  */
 160 static int emitted_frame_related_regs[number_of_ia64_frame_regs];
 161 \f
 162 static int ia64_first_cycle_multipass_dfa_lookahead (void);
 163 static void ia64_dependencies_evaluation_hook (rtx_insn *, rtx_insn *);
 164 static void ia64_init_dfa_pre_cycle_insn (void);
 165 static rtx ia64_dfa_pre_cycle_insn (void);
 166 static int ia64_first_cycle_multipass_dfa_lookahead_guard (rtx_insn *, int);
 167 static int ia64_dfa_new_cycle (FILE *, int, rtx_insn *, int, int, int *);
 168 static void ia64_h_i_d_extended (void);
 169 static void * ia64_alloc_sched_context (void);
 170 static void ia64_init_sched_context (void *, bool);
 171 static void ia64_set_sched_context (void *);
 172 static void ia64_clear_sched_context (void *);
 173 static void ia64_free_sched_context (void *);
 174 static int ia64_mode_to_int (machine_mode);
 175 static void ia64_set_sched_flags (spec_info_t);
 176 static ds_t ia64_get_insn_spec_ds (rtx_insn *);
 177 static ds_t ia64_get_insn_checked_ds (rtx_insn *);
 178 static bool ia64_skip_rtx_p (const_rtx);
 179 static int ia64_speculate_insn (rtx_insn *, ds_t, rtx *);
 180 static bool ia64_needs_block_p (ds_t);
 181 static rtx ia64_gen_spec_check (rtx_insn *, rtx_insn *, ds_t);
 182 static int ia64_spec_check_p (rtx);
 183 static int ia64_spec_check_src_p (rtx);
 184 static rtx gen_tls_get_addr (void);
 185 static rtx gen_thread_pointer (void);
 186 static int find_gr_spill (enum ia64_frame_regs, int);
 187 static int next_scratch_gr_reg (void);
 188 static void mark_reg_gr_used_mask (rtx, void *);
 189 static void ia64_compute_frame_size (HOST_WIDE_INT);
 190 static void setup_spill_pointers (int, rtx, HOST_WIDE_INT);
 191 static void finish_spill_pointers (void);
 192 static rtx spill_restore_mem (rtx, HOST_WIDE_INT);
 193 static void do_spill (rtx (*)(rtx, rtx, rtx), rtx, HOST_WIDE_INT, rtx);
 194 static void do_restore (rtx (*)(rtx, rtx, rtx), rtx, HOST_WIDE_INT);
 195 static rtx gen_movdi_x (rtx, rtx, rtx);
 196 static rtx gen_fr_spill_x (rtx, rtx, rtx);
 197 static rtx gen_fr_restore_x (rtx, rtx, rtx);
 198
 199 static void ia64_option_override (void);
 200 static bool ia64_can_eliminate (const int, const int);
 201 static machine_mode hfa_element_mode (const_tree, bool);
 202 static void ia64_setup_incoming_varargs (cumulative_args_t,
 203                                          const function_arg_info &,
 204                                          int *, int);
 205 static int ia64_arg_partial_bytes (cumulative_args_t,
 206                                    const function_arg_info &);
 207 static rtx ia64_function_arg (cumulative_args_t, const function_arg_info &);
 208 static rtx ia64_function_incoming_arg (cumulative_args_t,
 209                                        const function_arg_info &);
 210 static void ia64_function_arg_advance (cumulative_args_t, machine_mode,
 211                                        const_tree, bool);
 212 static pad_direction ia64_function_arg_padding (machine_mode, const_tree);
 213 static unsigned int ia64_function_arg_boundary (machine_mode,
 214                                                 const_tree);
 215 static bool ia64_function_ok_for_sibcall (tree, tree);
 216 static bool ia64_return_in_memory (const_tree, const_tree);
 217 static rtx ia64_function_value (const_tree, const_tree, bool);
 218 static rtx ia64_libcall_value (machine_mode, const_rtx);
 219 static bool ia64_function_value_regno_p (const unsigned int);
 220 static int ia64_register_move_cost (machine_mode, reg_class_t,
 221                                     reg_class_t);
 222 static int ia64_memory_move_cost (machine_mode mode, reg_class_t,
 223                                   bool);
 224 static bool ia64_rtx_costs (rtx, machine_mode, int, int, int *, bool);
 225 static int ia64_unspec_may_trap_p (const_rtx, unsigned);
 226 static void fix_range (const char *);
 227 static struct machine_function * ia64_init_machine_status (void);
 228 static void emit_insn_group_barriers (FILE *);
 229 static void emit_all_insn_group_barriers (FILE *);
 230 static void final_emit_insn_group_barriers (FILE *);
 231 static void emit_predicate_relation_info (void);
 232 static void ia64_reorg (void);
 233 static bool ia64_in_small_data_p (const_tree);
 234 static void process_epilogue (FILE *, rtx, bool, bool);
 235
 236 static bool ia64_assemble_integer (rtx, unsigned int, int);
 237 static void ia64_output_function_prologue (FILE *);
 238 static void ia64_output_function_epilogue (FILE *);
 239 static void ia64_output_function_end_prologue (FILE *);
 240
 241 static void ia64_print_operand (FILE *, rtx, int);
 242 static void ia64_print_operand_address (FILE *, machine_mode, rtx);
 243 static bool ia64_print_operand_punct_valid_p (unsigned char code);
 244
 245 static int ia64_issue_rate (void);
 246 static int ia64_adjust_cost (rtx_insn *, int, rtx_insn *, int, dw_t);
 247 static void ia64_sched_init (FILE *, int, int);
 248 static void ia64_sched_init_global (FILE *, int, int);
 249 static void ia64_sched_finish_global (FILE *, int);
 250 static void ia64_sched_finish (FILE *, int);
 251 static int ia64_dfa_sched_reorder (FILE *, int, rtx_insn **, int *, int, int);
 252 static int ia64_sched_reorder (FILE *, int, rtx_insn **, int *, int);
 253 static int ia64_sched_reorder2 (FILE *, int, rtx_insn **, int *, int);
 254 static int ia64_variable_issue (FILE *, int, rtx_insn *, int);
 255
 256 static void ia64_asm_unwind_emit (FILE *, rtx_insn *);
 257 static void ia64_asm_emit_except_personality (rtx);
 258 static void ia64_asm_init_sections (void);
 259
 260 static enum unwind_info_type ia64_debug_unwind_info (void);
 261
 262 static struct bundle_state *get_free_bundle_state (void);
 263 static void free_bundle_state (struct bundle_state *);
 264 static void initiate_bundle_states (void);
 265 static void finish_bundle_states (void);
 266 static int insert_bundle_state (struct bundle_state *);
 267 static void initiate_bundle_state_table (void);
 268 static void finish_bundle_state_table (void);
 269 static int try_issue_nops (struct bundle_state *, int);
 270 static int try_issue_insn (struct bundle_state *, rtx);
 271 static void issue_nops_and_insn (struct bundle_state *, int, rtx_insn *,
 272                                  int, int);
 273 static int get_max_pos (state_t);
 274 static int get_template (state_t, int);
 275
 276 static rtx_insn *get_next_important_insn (rtx_insn *, rtx_insn *);
 277 static bool important_for_bundling_p (rtx_insn *);
 278 static bool unknown_for_bundling_p (rtx_insn *);
 279 static void bundling (FILE *, int, rtx_insn *, rtx_insn *);
 280
 281 static void ia64_output_mi_thunk (FILE *, tree, HOST_WIDE_INT,
 282                                   HOST_WIDE_INT, tree);
 283 static void ia64_file_start (void);
 284 static void ia64_globalize_decl_name (FILE *, tree);
 285
 286 static int ia64_hpux_reloc_rw_mask (void) ATTRIBUTE_UNUSED;
 287 static int ia64_reloc_rw_mask (void) ATTRIBUTE_UNUSED;
 288 static section *ia64_select_rtx_section (machine_mode, rtx,
 289                                          unsigned HOST_WIDE_INT);
 290 static void ia64_output_dwarf_dtprel (FILE *, int, rtx)
 291      ATTRIBUTE_UNUSED;
 292 static unsigned int ia64_section_type_flags (tree, const char *, int);
 293 static void ia64_init_libfuncs (void)
 294      ATTRIBUTE_UNUSED;
 295 static void ia64_hpux_init_libfuncs (void)
 296      ATTRIBUTE_UNUSED;
 297 static void ia64_sysv4_init_libfuncs (void)
 298      ATTRIBUTE_UNUSED;
 299 static void ia64_vms_init_libfuncs (void)
 300      ATTRIBUTE_UNUSED;
 301 static void ia64_soft_fp_init_libfuncs (void)
 302      ATTRIBUTE_UNUSED;
 303 static bool ia64_vms_valid_pointer_mode (scalar_int_mode mode)
 304      ATTRIBUTE_UNUSED;
 305 static tree ia64_vms_common_object_attribute (tree *, tree, tree, int, bool *)
 306      ATTRIBUTE_UNUSED;
 307
 308 static bool ia64_attribute_takes_identifier_p (const_tree);
 309 static tree ia64_handle_model_attribute (tree *, tree, tree, int, bool *);
 310 static tree ia64_handle_version_id_attribute (tree *, tree, tree, int, bool *);
 311 static void ia64_encode_section_info (tree, rtx, int);
 312 static rtx ia64_struct_value_rtx (tree, int);
 313 static tree ia64_gimplify_va_arg (tree, tree, gimple_seq *, gimple_seq *);
 314 static bool ia64_scalar_mode_supported_p (scalar_mode mode);
 315 static bool ia64_vector_mode_supported_p (machine_mode mode);
 316 static bool ia64_legitimate_constant_p (machine_mode, rtx);
 317 static bool ia64_legitimate_address_p (machine_mode, rtx, bool);
 318 static bool ia64_cannot_force_const_mem (machine_mode, rtx);
 319 static const char *ia64_mangle_type (const_tree);
 320 static const char *ia64_invalid_conversion (const_tree, const_tree);
 321 static const char *ia64_invalid_unary_op (int, const_tree);
 322 static const char *ia64_invalid_binary_op (int, const_tree, const_tree);
 323 static machine_mode ia64_c_mode_for_suffix (char);
 324 static void ia64_trampoline_init (rtx, tree, rtx);
 325 static void ia64_override_options_after_change (void);
 326 static bool ia64_member_type_forces_blk (const_tree, machine_mode);
 327
 328 static tree ia64_fold_builtin (tree, int, tree *, bool);
 329 static tree ia64_builtin_decl (unsigned, bool);
 330
 331 static reg_class_t ia64_preferred_reload_class (rtx, reg_class_t);
 332 static fixed_size_mode ia64_get_reg_raw_mode (int regno);
 333 static section * ia64_hpux_function_section (tree, enum node_frequency,
 334                                              bool, bool);
 335
 336 static bool ia64_vectorize_vec_perm_const (machine_mode, rtx, rtx, rtx,
 337                                            const vec_perm_indices &);
 338
 339 static unsigned int ia64_hard_regno_nregs (unsigned int, machine_mode);
 340 static bool ia64_hard_regno_mode_ok (unsigned int, machine_mode);
 341 static bool ia64_modes_tieable_p (machine_mode, machine_mode);
 342 static bool ia64_can_change_mode_class (machine_mode, machine_mode,
 343                                         reg_class_t);
 344
 345 #define MAX_VECT_LEN    8
 346
 347 struct expand_vec_perm_d
 348 {
 349   rtx target, op0, op1;
 350   unsigned char perm[MAX_VECT_LEN];
 351   machine_mode vmode;
 352   unsigned char nelt;
 353   bool one_operand_p;
 354   bool testing_p;
 355 };
 356
 357 static bool ia64_expand_vec_perm_const_1 (struct expand_vec_perm_d *d);
 358
 359 \f
 360 /* Table of valid machine attributes.  */
 361 static const struct attribute_spec ia64_attribute_table[] =
 362 {
 363   /* { name, min_len, max_len, decl_req, type_req, fn_type_req,
 364        affects_type_identity, handler, exclude } */
 365   { "syscall_linkage", 0, 0, false, true,  true,  false, NULL, NULL },
 366   { "model",           1, 1, true, false, false,  false,
 367     ia64_handle_model_attribute, NULL },
 368 #if TARGET_ABI_OPEN_VMS
 369   { "common_object",   1, 1, true, false, false, false,
 370     ia64_vms_common_object_attribute, NULL },
 371 #endif
 372   { "version_id",      1, 1, true, false, false, false,
 373     ia64_handle_version_id_attribute, NULL },
 374   { NULL,              0, 0, false, false, false, false, NULL, NULL }
 375 };
 376
 377 /* Initialize the GCC target structure.  */
 378 #undef TARGET_ATTRIBUTE_TABLE
 379 #define TARGET_ATTRIBUTE_TABLE ia64_attribute_table
 380
 381 #undef TARGET_INIT_BUILTINS
 382 #define TARGET_INIT_BUILTINS ia64_init_builtins
 383
 384 #undef TARGET_FOLD_BUILTIN
 385 #define TARGET_FOLD_BUILTIN ia64_fold_builtin
 386
 387 #undef TARGET_EXPAND_BUILTIN
 388 #define TARGET_EXPAND_BUILTIN ia64_expand_builtin
 389
 390 #undef TARGET_BUILTIN_DECL
 391 #define TARGET_BUILTIN_DECL ia64_builtin_decl
 392
 393 #undef TARGET_ASM_BYTE_OP
 394 #define TARGET_ASM_BYTE_OP "\tdata1\t"
 395 #undef TARGET_ASM_ALIGNED_HI_OP
 396 #define TARGET_ASM_ALIGNED_HI_OP "\tdata2\t"
 397 #undef TARGET_ASM_ALIGNED_SI_OP
 398 #define TARGET_ASM_ALIGNED_SI_OP "\tdata4\t"
 399 #undef TARGET_ASM_ALIGNED_DI_OP
 400 #define TARGET_ASM_ALIGNED_DI_OP "\tdata8\t"
 401 #undef TARGET_ASM_UNALIGNED_HI_OP
 402 #define TARGET_ASM_UNALIGNED_HI_OP "\tdata2.ua\t"
 403 #undef TARGET_ASM_UNALIGNED_SI_OP
 404 #define TARGET_ASM_UNALIGNED_SI_OP "\tdata4.ua\t"
 405 #undef TARGET_ASM_UNALIGNED_DI_OP
 406 #define TARGET_ASM_UNALIGNED_DI_OP "\tdata8.ua\t"
 407 #undef TARGET_ASM_INTEGER
 408 #define TARGET_ASM_INTEGER ia64_assemble_integer
 409
 410 #undef TARGET_OPTION_OVERRIDE
 411 #define TARGET_OPTION_OVERRIDE ia64_option_override
 412
 413 #undef TARGET_ASM_FUNCTION_PROLOGUE
 414 #define TARGET_ASM_FUNCTION_PROLOGUE ia64_output_function_prologue
 415 #undef TARGET_ASM_FUNCTION_END_PROLOGUE
 416 #define TARGET_ASM_FUNCTION_END_PROLOGUE ia64_output_function_end_prologue
 417 #undef TARGET_ASM_FUNCTION_EPILOGUE
 418 #define TARGET_ASM_FUNCTION_EPILOGUE ia64_output_function_epilogue
 419
 420 #undef TARGET_PRINT_OPERAND
 421 #define TARGET_PRINT_OPERAND ia64_print_operand
 422 #undef TARGET_PRINT_OPERAND_ADDRESS
 423 #define TARGET_PRINT_OPERAND_ADDRESS ia64_print_operand_address
 424 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
 425 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ia64_print_operand_punct_valid_p
 426
 427 #undef TARGET_IN_SMALL_DATA_P
 428 #define TARGET_IN_SMALL_DATA_P  ia64_in_small_data_p
 429
 430 #undef TARGET_SCHED_ADJUST_COST
 431 #define TARGET_SCHED_ADJUST_COST ia64_adjust_cost
 432 #undef TARGET_SCHED_ISSUE_RATE
 433 #define TARGET_SCHED_ISSUE_RATE ia64_issue_rate
 434 #undef TARGET_SCHED_VARIABLE_ISSUE
 435 #define TARGET_SCHED_VARIABLE_ISSUE ia64_variable_issue
 436 #undef TARGET_SCHED_INIT
 437 #define TARGET_SCHED_INIT ia64_sched_init
 438 #undef TARGET_SCHED_FINISH
 439 #define TARGET_SCHED_FINISH ia64_sched_finish
 440 #undef TARGET_SCHED_INIT_GLOBAL
 441 #define TARGET_SCHED_INIT_GLOBAL ia64_sched_init_global
 442 #undef TARGET_SCHED_FINISH_GLOBAL
 443 #define TARGET_SCHED_FINISH_GLOBAL ia64_sched_finish_global
 444 #undef TARGET_SCHED_REORDER
 445 #define TARGET_SCHED_REORDER ia64_sched_reorder
 446 #undef TARGET_SCHED_REORDER2
 447 #define TARGET_SCHED_REORDER2 ia64_sched_reorder2
 448
 449 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
 450 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK ia64_dependencies_evaluation_hook
 451
 452 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
 453 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD ia64_first_cycle_multipass_dfa_lookahead
 454
 455 #undef TARGET_SCHED_INIT_DFA_PRE_CYCLE_INSN
 456 #define TARGET_SCHED_INIT_DFA_PRE_CYCLE_INSN ia64_init_dfa_pre_cycle_insn
 457 #undef TARGET_SCHED_DFA_PRE_CYCLE_INSN
 458 #define TARGET_SCHED_DFA_PRE_CYCLE_INSN ia64_dfa_pre_cycle_insn
 459
 460 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
 461 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD\
 462   ia64_first_cycle_multipass_dfa_lookahead_guard
 463
 464 #undef TARGET_SCHED_DFA_NEW_CYCLE
 465 #define TARGET_SCHED_DFA_NEW_CYCLE ia64_dfa_new_cycle
 466
 467 #undef TARGET_SCHED_H_I_D_EXTENDED
 468 #define TARGET_SCHED_H_I_D_EXTENDED ia64_h_i_d_extended
 469
 470 #undef TARGET_SCHED_ALLOC_SCHED_CONTEXT
 471 #define TARGET_SCHED_ALLOC_SCHED_CONTEXT ia64_alloc_sched_context
 472
 473 #undef TARGET_SCHED_INIT_SCHED_CONTEXT
 474 #define TARGET_SCHED_INIT_SCHED_CONTEXT ia64_init_sched_context
 475
 476 #undef TARGET_SCHED_SET_SCHED_CONTEXT
 477 #define TARGET_SCHED_SET_SCHED_CONTEXT ia64_set_sched_context
 478
 479 #undef TARGET_SCHED_CLEAR_SCHED_CONTEXT
 480 #define TARGET_SCHED_CLEAR_SCHED_CONTEXT ia64_clear_sched_context
 481
 482 #undef TARGET_SCHED_FREE_SCHED_CONTEXT
 483 #define TARGET_SCHED_FREE_SCHED_CONTEXT ia64_free_sched_context
 484
 485 #undef TARGET_SCHED_SET_SCHED_FLAGS
 486 #define TARGET_SCHED_SET_SCHED_FLAGS ia64_set_sched_flags
 487
 488 #undef TARGET_SCHED_GET_INSN_SPEC_DS
 489 #define TARGET_SCHED_GET_INSN_SPEC_DS ia64_get_insn_spec_ds
 490
 491 #undef TARGET_SCHED_GET_INSN_CHECKED_DS
 492 #define TARGET_SCHED_GET_INSN_CHECKED_DS ia64_get_insn_checked_ds
 493
 494 #undef TARGET_SCHED_SPECULATE_INSN
 495 #define TARGET_SCHED_SPECULATE_INSN ia64_speculate_insn
 496
 497 #undef TARGET_SCHED_NEEDS_BLOCK_P
 498 #define TARGET_SCHED_NEEDS_BLOCK_P ia64_needs_block_p
 499
 500 #undef TARGET_SCHED_GEN_SPEC_CHECK
 501 #define TARGET_SCHED_GEN_SPEC_CHECK ia64_gen_spec_check
 502
 503 #undef TARGET_SCHED_SKIP_RTX_P
 504 #define TARGET_SCHED_SKIP_RTX_P ia64_skip_rtx_p
 505
 506 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
 507 #define TARGET_FUNCTION_OK_FOR_SIBCALL ia64_function_ok_for_sibcall
 508 #undef TARGET_ARG_PARTIAL_BYTES
 509 #define TARGET_ARG_PARTIAL_BYTES ia64_arg_partial_bytes
 510 #undef TARGET_FUNCTION_ARG
 511 #define TARGET_FUNCTION_ARG ia64_function_arg
 512 #undef TARGET_FUNCTION_INCOMING_ARG
 513 #define TARGET_FUNCTION_INCOMING_ARG ia64_function_incoming_arg
 514 #undef TARGET_FUNCTION_ARG_ADVANCE
 515 #define TARGET_FUNCTION_ARG_ADVANCE ia64_function_arg_advance
 516 #undef TARGET_FUNCTION_ARG_PADDING
 517 #define TARGET_FUNCTION_ARG_PADDING ia64_function_arg_padding
 518 #undef TARGET_FUNCTION_ARG_BOUNDARY
 519 #define TARGET_FUNCTION_ARG_BOUNDARY ia64_function_arg_boundary
 520
 521 #undef TARGET_ASM_OUTPUT_MI_THUNK
 522 #define TARGET_ASM_OUTPUT_MI_THUNK ia64_output_mi_thunk
 523 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
 524 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK hook_bool_const_tree_hwi_hwi_const_tree_true
 525
 526 #undef TARGET_ASM_FILE_START
 527 #define TARGET_ASM_FILE_START ia64_file_start
 528
 529 #undef TARGET_ASM_GLOBALIZE_DECL_NAME
 530 #define TARGET_ASM_GLOBALIZE_DECL_NAME ia64_globalize_decl_name
 531
 532 #undef TARGET_REGISTER_MOVE_COST
 533 #define TARGET_REGISTER_MOVE_COST ia64_register_move_cost
 534 #undef TARGET_MEMORY_MOVE_COST
 535 #define TARGET_MEMORY_MOVE_COST ia64_memory_move_cost
 536 #undef TARGET_RTX_COSTS
 537 #define TARGET_RTX_COSTS ia64_rtx_costs
 538 #undef TARGET_ADDRESS_COST
 539 #define TARGET_ADDRESS_COST hook_int_rtx_mode_as_bool_0
 540
 541 #undef TARGET_UNSPEC_MAY_TRAP_P
 542 #define TARGET_UNSPEC_MAY_TRAP_P ia64_unspec_may_trap_p
 543
 544 #undef TARGET_MACHINE_DEPENDENT_REORG
 545 #define TARGET_MACHINE_DEPENDENT_REORG ia64_reorg
 546
 547 #undef TARGET_ENCODE_SECTION_INFO
 548 #define TARGET_ENCODE_SECTION_INFO ia64_encode_section_info
 549
 550 #undef  TARGET_SECTION_TYPE_FLAGS
 551 #define TARGET_SECTION_TYPE_FLAGS  ia64_section_type_flags
 552
 553 #ifdef HAVE_AS_TLS
 554 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
 555 #define TARGET_ASM_OUTPUT_DWARF_DTPREL ia64_output_dwarf_dtprel
 556 #endif
 557
 558 /* ??? Investigate.  */
 559 #if 0
 560 #undef TARGET_PROMOTE_PROTOTYPES
 561 #define TARGET_PROMOTE_PROTOTYPES hook_bool_tree_true
 562 #endif
 563
 564 #undef TARGET_FUNCTION_VALUE
 565 #define TARGET_FUNCTION_VALUE ia64_function_value
 566 #undef TARGET_LIBCALL_VALUE
 567 #define TARGET_LIBCALL_VALUE ia64_libcall_value
 568 #undef TARGET_FUNCTION_VALUE_REGNO_P
 569 #define TARGET_FUNCTION_VALUE_REGNO_P ia64_function_value_regno_p
 570
 571 #undef TARGET_STRUCT_VALUE_RTX
 572 #define TARGET_STRUCT_VALUE_RTX ia64_struct_value_rtx
 573 #undef TARGET_RETURN_IN_MEMORY
 574 #define TARGET_RETURN_IN_MEMORY ia64_return_in_memory
 575 #undef TARGET_SETUP_INCOMING_VARARGS
 576 #define TARGET_SETUP_INCOMING_VARARGS ia64_setup_incoming_varargs
 577 #undef TARGET_STRICT_ARGUMENT_NAMING
 578 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
 579 #undef TARGET_MUST_PASS_IN_STACK
 580 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
 581 #undef TARGET_GET_RAW_RESULT_MODE
 582 #define TARGET_GET_RAW_RESULT_MODE ia64_get_reg_raw_mode
 583 #undef TARGET_GET_RAW_ARG_MODE
 584 #define TARGET_GET_RAW_ARG_MODE ia64_get_reg_raw_mode
 585
 586 #undef TARGET_MEMBER_TYPE_FORCES_BLK
 587 #define TARGET_MEMBER_TYPE_FORCES_BLK ia64_member_type_forces_blk
 588
 589 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
 590 #define TARGET_GIMPLIFY_VA_ARG_EXPR ia64_gimplify_va_arg
 591
 592 #undef TARGET_ASM_UNWIND_EMIT
 593 #define TARGET_ASM_UNWIND_EMIT ia64_asm_unwind_emit
 594 #undef TARGET_ASM_EMIT_EXCEPT_PERSONALITY
 595 #define TARGET_ASM_EMIT_EXCEPT_PERSONALITY  ia64_asm_emit_except_personality
 596 #undef TARGET_ASM_INIT_SECTIONS
 597 #define TARGET_ASM_INIT_SECTIONS  ia64_asm_init_sections
 598
 599 #undef TARGET_DEBUG_UNWIND_INFO
 600 #define TARGET_DEBUG_UNWIND_INFO  ia64_debug_unwind_info
 601
 602 #undef TARGET_SCALAR_MODE_SUPPORTED_P
 603 #define TARGET_SCALAR_MODE_SUPPORTED_P ia64_scalar_mode_supported_p
 604 #undef TARGET_VECTOR_MODE_SUPPORTED_P
 605 #define TARGET_VECTOR_MODE_SUPPORTED_P ia64_vector_mode_supported_p
 606
 607 #undef TARGET_LEGITIMATE_CONSTANT_P
 608 #define TARGET_LEGITIMATE_CONSTANT_P ia64_legitimate_constant_p
 609 #undef TARGET_LEGITIMATE_ADDRESS_P
 610 #define TARGET_LEGITIMATE_ADDRESS_P ia64_legitimate_address_p
 611
 612 #undef TARGET_LRA_P
 613 #define TARGET_LRA_P hook_bool_void_false
 614
 615 #undef TARGET_CANNOT_FORCE_CONST_MEM
 616 #define TARGET_CANNOT_FORCE_CONST_MEM ia64_cannot_force_const_mem
 617
 618 #undef TARGET_MANGLE_TYPE
 619 #define TARGET_MANGLE_TYPE ia64_mangle_type
 620
 621 #undef TARGET_INVALID_CONVERSION
 622 #define TARGET_INVALID_CONVERSION ia64_invalid_conversion
 623 #undef TARGET_INVALID_UNARY_OP
 624 #define TARGET_INVALID_UNARY_OP ia64_invalid_unary_op
 625 #undef TARGET_INVALID_BINARY_OP
 626 #define TARGET_INVALID_BINARY_OP ia64_invalid_binary_op
 627
 628 #undef TARGET_C_MODE_FOR_SUFFIX
 629 #define TARGET_C_MODE_FOR_SUFFIX ia64_c_mode_for_suffix
 630
 631 #undef TARGET_CAN_ELIMINATE
 632 #define TARGET_CAN_ELIMINATE ia64_can_eliminate
 633
 634 #undef TARGET_TRAMPOLINE_INIT
 635 #define TARGET_TRAMPOLINE_INIT ia64_trampoline_init
 636
 637 #undef TARGET_CAN_USE_DOLOOP_P
 638 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
 639 #undef TARGET_INVALID_WITHIN_DOLOOP
 640 #define TARGET_INVALID_WITHIN_DOLOOP hook_constcharptr_const_rtx_insn_null
 641
 642 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
 643 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE ia64_override_options_after_change
 644
 645 #undef TARGET_PREFERRED_RELOAD_CLASS
 646 #define TARGET_PREFERRED_RELOAD_CLASS ia64_preferred_reload_class
 647
 648 #undef TARGET_DELAY_SCHED2
 649 #define TARGET_DELAY_SCHED2 true
 650
 651 /* Variable tracking should be run after all optimizations which
 652    change order of insns.  It also needs a valid CFG.  */
 653 #undef TARGET_DELAY_VARTRACK
 654 #define TARGET_DELAY_VARTRACK true
 655
 656 #undef TARGET_VECTORIZE_VEC_PERM_CONST
 657 #define TARGET_VECTORIZE_VEC_PERM_CONST ia64_vectorize_vec_perm_const
 658
 659 #undef TARGET_ATTRIBUTE_TAKES_IDENTIFIER_P
 660 #define TARGET_ATTRIBUTE_TAKES_IDENTIFIER_P ia64_attribute_takes_identifier_p
 661
 662 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
 663 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 0
 664
 665 #undef TARGET_HARD_REGNO_NREGS
 666 #define TARGET_HARD_REGNO_NREGS ia64_hard_regno_nregs
 667 #undef TARGET_HARD_REGNO_MODE_OK
 668 #define TARGET_HARD_REGNO_MODE_OK ia64_hard_regno_mode_ok
 669
 670 #undef TARGET_MODES_TIEABLE_P
 671 #define TARGET_MODES_TIEABLE_P ia64_modes_tieable_p
 672
 673 #undef TARGET_CAN_CHANGE_MODE_CLASS
 674 #define TARGET_CAN_CHANGE_MODE_CLASS ia64_can_change_mode_class
 675
 676 #undef TARGET_CONSTANT_ALIGNMENT
 677 #define TARGET_CONSTANT_ALIGNMENT constant_alignment_word_strings
 678
 679 struct gcc_target targetm = TARGET_INITIALIZER;
 680 \f
 681 /* Returns TRUE iff the target attribute indicated by ATTR_ID takes a plain
 682    identifier as an argument, so the front end shouldn't look it up.  */
 683
 684 static bool
 685 ia64_attribute_takes_identifier_p (const_tree attr_id)
 686 {
 687   if (is_attribute_p ("model", attr_id))
 688     return true;
 689 #if TARGET_ABI_OPEN_VMS
 690   if (is_attribute_p ("common_object", attr_id))
 691     return true;
 692 #endif
 693   return false;
 694 }
 695
 696 typedef enum
 697   {
 698     ADDR_AREA_NORMAL,   /* normal address area */
 699     ADDR_AREA_SMALL     /* addressable by "addl" (-2MB < addr < 2MB) */
 700   }
 701 ia64_addr_area;
 702
 703 static GTY(()) tree small_ident1;
 704 static GTY(()) tree small_ident2;
 705
 706 static void
 707 init_idents (void)
 708 {
 709   if (small_ident1 == 0)
 710     {
 711       small_ident1 = get_identifier ("small");
 712       small_ident2 = get_identifier ("__small__");
 713     }
 714 }
 715
 716 /* Retrieve the address area that has been chosen for the given decl.  */
 717
 718 static ia64_addr_area
 719 ia64_get_addr_area (tree decl)
 720 {
 721   tree model_attr;
 722
 723   model_attr = lookup_attribute ("model", DECL_ATTRIBUTES (decl));
 724   if (model_attr)
 725     {
 726       tree id;
 727
 728       init_idents ();
 729       id = TREE_VALUE (TREE_VALUE (model_attr));
 730       if (id == small_ident1 || id == small_ident2)
 731         return ADDR_AREA_SMALL;
 732     }
 733   return ADDR_AREA_NORMAL;
 734 }
 735
 736 static tree
 737 ia64_handle_model_attribute (tree *node, tree name, tree args,
 738                              int flags ATTRIBUTE_UNUSED, bool *no_add_attrs)
 739 {
 740   ia64_addr_area addr_area = ADDR_AREA_NORMAL;
 741   ia64_addr_area area;
 742   tree arg, decl = *node;
 743
 744   init_idents ();
 745   arg = TREE_VALUE (args);
 746   if (arg == small_ident1 || arg == small_ident2)
 747     {
 748       addr_area = ADDR_AREA_SMALL;
 749     }
 750   else
 751     {
 752       warning (OPT_Wattributes, "invalid argument of %qE attribute",
 753                name);
 754       *no_add_attrs = true;
 755     }
 756
 757   switch (TREE_CODE (decl))
 758     {
 759     case VAR_DECL:
 760       if ((DECL_CONTEXT (decl) && TREE_CODE (DECL_CONTEXT (decl))
 761            == FUNCTION_DECL)
 762           && !TREE_STATIC (decl))
 763         {
 764           error_at (DECL_SOURCE_LOCATION (decl),
 765                     "an address area attribute cannot be specified for "
 766                     "local variables");
 767           *no_add_attrs = true;
 768         }
 769       area = ia64_get_addr_area (decl);
 770       if (area != ADDR_AREA_NORMAL && addr_area != area)
 771         {
 772           error ("address area of %q+D conflicts with previous "
 773                  "declaration", decl);
 774           *no_add_attrs = true;
 775         }
 776       break;
 777
 778     case FUNCTION_DECL:
 779       error_at (DECL_SOURCE_LOCATION (decl),
 780                 "address area attribute cannot be specified for "
 781                 "functions");
 782       *no_add_attrs = true;
 783       break;
 784
 785     default:
 786       warning (OPT_Wattributes, "%qE attribute ignored",
 787                name);
 788       *no_add_attrs = true;
 789       break;
 790     }
 791
 792   return NULL_TREE;
 793 }
 794
 795 /* Part of the low level implementation of DEC Ada pragma Common_Object which
 796    enables the shared use of variables stored in overlaid linker areas
 797    corresponding to the use of Fortran COMMON.  */
 798
 799 static tree
 800 ia64_vms_common_object_attribute (tree *node, tree name, tree args,
 801                                   int flags ATTRIBUTE_UNUSED,
 802                                   bool *no_add_attrs)
 803 {
 804     tree decl = *node;
 805     tree id;
 806
 807     gcc_assert (DECL_P (decl));
 808
 809     DECL_COMMON (decl) = 1;
 810     id = TREE_VALUE (args);
 811     if (TREE_CODE (id) != IDENTIFIER_NODE && TREE_CODE (id) != STRING_CST)
 812       {
 813         error ("%qE attribute requires a string constant argument", name);
 814         *no_add_attrs = true;
 815         return NULL_TREE;
 816       }
 817     return NULL_TREE;
 818 }
 819
 820 /* Part of the low level implementation of DEC Ada pragma Common_Object.  */
 821
 822 void
 823 ia64_vms_output_aligned_decl_common (FILE *file, tree decl, const char *name,
 824                                      unsigned HOST_WIDE_INT size,
 825                                      unsigned int align)
 826 {
 827   tree attr = DECL_ATTRIBUTES (decl);
 828
 829   if (attr)
 830     attr = lookup_attribute ("common_object", attr);
 831   if (attr)
 832     {
 833       tree id = TREE_VALUE (TREE_VALUE (attr));
 834       const char *name;
 835
 836       if (TREE_CODE (id) == IDENTIFIER_NODE)
 837         name = IDENTIFIER_POINTER (id);
 838       else if (TREE_CODE (id) == STRING_CST)
 839         name = TREE_STRING_POINTER (id);
 840       else
 841         abort ();
 842
 843       fprintf (file, "\t.vms_common\t\"%s\",", name);
 844     }
 845   else
 846     fprintf (file, "%s", COMMON_ASM_OP);
 847
 848   /*  Code from elfos.h.  */
 849   assemble_name (file, name);
 850   fprintf (file, "," HOST_WIDE_INT_PRINT_UNSIGNED",%u",
 851            size, align / BITS_PER_UNIT);
 852
 853   fputc ('\n', file);
 854 }
 855
 856 static void
 857 ia64_encode_addr_area (tree decl, rtx symbol)
 858 {
 859   int flags;
 860
 861   flags = SYMBOL_REF_FLAGS (symbol);
 862   switch (ia64_get_addr_area (decl))
 863     {
 864     case ADDR_AREA_NORMAL: break;
 865     case ADDR_AREA_SMALL: flags |= SYMBOL_FLAG_SMALL_ADDR; break;
 866     default: gcc_unreachable ();
 867     }
 868   SYMBOL_REF_FLAGS (symbol) = flags;
 869 }
 870
 871 static void
 872 ia64_encode_section_info (tree decl, rtx rtl, int first)
 873 {
 874   default_encode_section_info (decl, rtl, first);
 875
 876   /* Careful not to prod global register variables.  */
 877   if (TREE_CODE (decl) == VAR_DECL
 878       && GET_CODE (DECL_RTL (decl)) == MEM
 879       && GET_CODE (XEXP (DECL_RTL (decl), 0)) == SYMBOL_REF
 880       && (TREE_STATIC (decl) || DECL_EXTERNAL (decl)))
 881     ia64_encode_addr_area (decl, XEXP (rtl, 0));
 882 }
 883 \f
 884 /* Return 1 if the operands of a move are ok.  */
 885
 886 int
 887 ia64_move_ok (rtx dst, rtx src)
 888 {
 889   /* If we're under init_recog_no_volatile, we'll not be able to use
 890      memory_operand.  So check the code directly and don't worry about
 891      the validity of the underlying address, which should have been
 892      checked elsewhere anyway.  */
 893   if (GET_CODE (dst) != MEM)
 894     return 1;
 895   if (GET_CODE (src) == MEM)
 896     return 0;
 897   if (register_operand (src, VOIDmode))
 898     return 1;
 899
 900   /* Otherwise, this must be a constant, and that either 0 or 0.0 or 1.0.  */
 901   if (INTEGRAL_MODE_P (GET_MODE (dst)))
 902     return src == const0_rtx;
 903   else
 904     return satisfies_constraint_G (src);
 905 }
 906
 907 /* Return 1 if the operands are ok for a floating point load pair.  */
 908
 909 int
 910 ia64_load_pair_ok (rtx dst, rtx src)
 911 {
 912   /* ??? There is a thinko in the implementation of the "x" constraint and the
 913      FP_REGS class.  The constraint will also reject (reg f30:TI) so we must
 914      also return false for it.  */
 915   if (GET_CODE (dst) != REG
 916       || !(FP_REGNO_P (REGNO (dst)) && FP_REGNO_P (REGNO (dst) + 1)))
 917     return 0;
 918   if (GET_CODE (src) != MEM || MEM_VOLATILE_P (src))
 919     return 0;
 920   switch (GET_CODE (XEXP (src, 0)))
 921     {
 922     case REG:
 923     case POST_INC:
 924       break;
 925     case POST_DEC:
 926       return 0;
 927     case POST_MODIFY:
 928       {
 929         rtx adjust = XEXP (XEXP (XEXP (src, 0), 1), 1);
 930
 931         if (GET_CODE (adjust) != CONST_INT
 932             || INTVAL (adjust) != GET_MODE_SIZE (GET_MODE (src)))
 933           return 0;
 934       }
 935       break;
 936     default:
 937       abort ();
 938     }
 939   return 1;
 940 }
 941
 942 int
 943 addp4_optimize_ok (rtx op1, rtx op2)
 944 {
 945   return (basereg_operand (op1, GET_MODE(op1)) !=
 946           basereg_operand (op2, GET_MODE(op2)));
 947 }
 948
 949 /* Check if OP is a mask suitable for use with SHIFT in a dep.z instruction.
 950    Return the length of the field, or <= 0 on failure.  */
 951
 952 int
 953 ia64_depz_field_mask (rtx rop, rtx rshift)
 954 {
 955   unsigned HOST_WIDE_INT op = INTVAL (rop);
 956   unsigned HOST_WIDE_INT shift = INTVAL (rshift);
 957
 958   /* Get rid of the zero bits we're shifting in.  */
 959   op >>= shift;
 960
 961   /* We must now have a solid block of 1's at bit 0.  */
 962   return exact_log2 (op + 1);
 963 }
 964
 965 /* Return the TLS model to use for ADDR.  */
 966
 967 static enum tls_model
 968 tls_symbolic_operand_type (rtx addr)
 969 {
 970   enum tls_model tls_kind = TLS_MODEL_NONE;
 971
 972   if (GET_CODE (addr) == CONST)
 973     {
 974       if (GET_CODE (XEXP (addr, 0)) == PLUS
 975           && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF)
 976         tls_kind = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (addr, 0), 0));
 977     }
 978   else if (GET_CODE (addr) == SYMBOL_REF)
 979     tls_kind = SYMBOL_REF_TLS_MODEL (addr);
 980
 981   return tls_kind;
 982 }
 983
 984 /* Returns true if REG (assumed to be a `reg' RTX) is valid for use
 985    as a base register.  */
 986
 987 static inline bool
 988 ia64_reg_ok_for_base_p (const_rtx reg, bool strict)
 989 {
 990   if (strict
 991       && REGNO_OK_FOR_BASE_P (REGNO (reg)))
 992     return true;
 993   else if (!strict
 994            && (GENERAL_REGNO_P (REGNO (reg))
 995                || !HARD_REGISTER_P (reg)))
 996     return true;
 997   else
 998     return false;
 999 }
1000
1001 static bool
1002 ia64_legitimate_address_reg (const_rtx reg, bool strict)
1003 {
1004   if ((REG_P (reg) && ia64_reg_ok_for_base_p (reg, strict))
1005       || (GET_CODE (reg) == SUBREG && REG_P (XEXP (reg, 0))
1006           && ia64_reg_ok_for_base_p (XEXP (reg, 0), strict)))
1007     return true;
1008
1009   return false;
1010 }
1011
1012 static bool
1013 ia64_legitimate_address_disp (const_rtx reg, const_rtx disp, bool strict)
1014 {
1015   if (GET_CODE (disp) == PLUS
1016       && rtx_equal_p (reg, XEXP (disp, 0))
1017       && (ia64_legitimate_address_reg (XEXP (disp, 1), strict)
1018           || (CONST_INT_P (XEXP (disp, 1))
1019               && IN_RANGE (INTVAL (XEXP (disp, 1)), -256, 255))))
1020     return true;
1021
1022   return false;
1023 }
1024
1025 /* Implement TARGET_LEGITIMATE_ADDRESS_P.  */
1026
1027 static bool
1028 ia64_legitimate_address_p (machine_mode mode ATTRIBUTE_UNUSED,
1029                            rtx x, bool strict)
1030 {
1031   if (ia64_legitimate_address_reg (x, strict))
1032     return true;
1033   else if ((GET_CODE (x) == POST_INC || GET_CODE (x) == POST_DEC)
1034            && ia64_legitimate_address_reg (XEXP (x, 0), strict)
1035            && XEXP (x, 0) != arg_pointer_rtx)
1036     return true;
1037   else if (GET_CODE (x) == POST_MODIFY
1038            && ia64_legitimate_address_reg (XEXP (x, 0), strict)
1039            && XEXP (x, 0) != arg_pointer_rtx
1040            && ia64_legitimate_address_disp (XEXP (x, 0), XEXP (x, 1), strict))
1041     return true;
1042   else
1043     return false;
1044 }
1045
1046 /* Return true if X is a constant that is valid for some immediate
1047    field in an instruction.  */
1048
1049 static bool
1050 ia64_legitimate_constant_p (machine_mode mode, rtx x)
1051 {
1052   switch (GET_CODE (x))
1053     {
1054     case CONST_INT:
1055     case LABEL_REF:
1056       return true;
1057
1058     case CONST_DOUBLE:
1059       if (GET_MODE (x) == VOIDmode || mode == SFmode || mode == DFmode)
1060         return true;
1061       return satisfies_constraint_G (x);
1062
1063     case CONST:
1064     case SYMBOL_REF:
1065       /* ??? Short term workaround for PR 28490.  We must make the code here
1066          match the code in ia64_expand_move and move_operand, even though they
1067          are both technically wrong.  */
1068       if (tls_symbolic_operand_type (x) == 0)
1069         {
1070           HOST_WIDE_INT addend = 0;
1071           rtx op = x;
1072
1073           if (GET_CODE (op) == CONST
1074               && GET_CODE (XEXP (op, 0)) == PLUS
1075               && GET_CODE (XEXP (XEXP (op, 0), 1)) == CONST_INT)
1076             {
1077               addend = INTVAL (XEXP (XEXP (op, 0), 1));
1078               op = XEXP (XEXP (op, 0), 0);
1079             }
1080
1081           if (any_offset_symbol_operand (op, mode)
1082               || function_operand (op, mode))
1083             return true;
1084           if (aligned_offset_symbol_operand (op, mode))
1085             return (addend & 0x3fff) == 0;
1086           return false;
1087         }
1088       return false;
1089
1090     case CONST_VECTOR:
1091       if (mode == V2SFmode)
1092         return satisfies_constraint_Y (x);
1093
1094       return (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
1095               && GET_MODE_SIZE (mode) <= 8);
1096
1097     default:
1098       return false;
1099     }
1100 }
1101
1102 /* Don't allow TLS addresses to get spilled to memory.  */
1103
1104 static bool
1105 ia64_cannot_force_const_mem (machine_mode mode, rtx x)
1106 {
1107   if (mode == RFmode)
1108     return true;
1109   return tls_symbolic_operand_type (x) != 0;
1110 }
1111
1112 /* Expand a symbolic constant load.  */
1113
1114 bool
1115 ia64_expand_load_address (rtx dest, rtx src)
1116 {
1117   gcc_assert (GET_CODE (dest) == REG);
1118
1119   /* ILP32 mode still loads 64-bits of data from the GOT.  This avoids
1120      having to pointer-extend the value afterward.  Other forms of address
1121      computation below are also more natural to compute as 64-bit quantities.
1122      If we've been given an SImode destination register, change it.  */
1123   if (GET_MODE (dest) != Pmode)
1124     dest = gen_rtx_REG_offset (dest, Pmode, REGNO (dest),
1125                                byte_lowpart_offset (Pmode, GET_MODE (dest)));
1126
1127   if (TARGET_NO_PIC)
1128     return false;
1129   if (small_addr_symbolic_operand (src, VOIDmode))
1130     return false;
1131
1132   if (TARGET_AUTO_PIC)
1133     emit_insn (gen_load_gprel64 (dest, src));
1134   else if (GET_CODE (src) == SYMBOL_REF && SYMBOL_REF_FUNCTION_P (src))
1135     emit_insn (gen_load_fptr (dest, src));
1136   else if (sdata_symbolic_operand (src, VOIDmode))
1137     emit_insn (gen_load_gprel (dest, src));
1138   else if (local_symbolic_operand64 (src, VOIDmode))
1139     {
1140       /* We want to use @gprel rather than @ltoff relocations for local
1141          symbols:
1142           - @gprel does not require dynamic linker
1143           - and does not use .sdata section
1144          https://gcc.gnu.org/bugzilla/60465 */
1145       emit_insn (gen_load_gprel64 (dest, src));
1146     }
1147   else
1148     {
1149       HOST_WIDE_INT addend = 0;
1150       rtx tmp;
1151
1152       /* We did split constant offsets in ia64_expand_move, and we did try
1153          to keep them split in move_operand, but we also allowed reload to
1154          rematerialize arbitrary constants rather than spill the value to
1155          the stack and reload it.  So we have to be prepared here to split
1156          them apart again.  */
1157       if (GET_CODE (src) == CONST)
1158         {
1159           HOST_WIDE_INT hi, lo;
1160
1161           hi = INTVAL (XEXP (XEXP (src, 0), 1));
1162           lo = ((hi & 0x3fff) ^ 0x2000) - 0x2000;
1163           hi = hi - lo;
1164
1165           if (lo != 0)
1166             {
1167               addend = lo;
1168               src = plus_constant (Pmode, XEXP (XEXP (src, 0), 0), hi);
1169             }
1170         }
1171
1172       tmp = gen_rtx_HIGH (Pmode, src);
1173       tmp = gen_rtx_PLUS (Pmode, tmp, pic_offset_table_rtx);
1174       emit_insn (gen_rtx_SET (dest, tmp));
1175
1176       tmp = gen_rtx_LO_SUM (Pmode, gen_const_mem (Pmode, dest), src);
1177       emit_insn (gen_rtx_SET (dest, tmp));
1178
1179       if (addend)
1180         {
1181           tmp = gen_rtx_PLUS (Pmode, dest, GEN_INT (addend));
1182           emit_insn (gen_rtx_SET (dest, tmp));
1183         }
1184     }
1185
1186   return true;
1187 }
1188
1189 static GTY(()) rtx gen_tls_tga;
1190 static rtx
1191 gen_tls_get_addr (void)
1192 {
1193   if (!gen_tls_tga)
1194     gen_tls_tga = init_one_libfunc ("__tls_get_addr");
1195   return gen_tls_tga;
1196 }
1197
1198 static GTY(()) rtx thread_pointer_rtx;
1199 static rtx
1200 gen_thread_pointer (void)
1201 {
1202   if (!thread_pointer_rtx)
1203     thread_pointer_rtx = gen_rtx_REG (Pmode, 13);
1204   return thread_pointer_rtx;
1205 }
1206
1207 static rtx
1208 ia64_expand_tls_address (enum tls_model tls_kind, rtx op0, rtx op1,
1209                          rtx orig_op1, HOST_WIDE_INT addend)
1210 {
1211   rtx tga_op1, tga_op2, tga_ret, tga_eqv, tmp;
1212   rtx_insn *insns;
1213   rtx orig_op0 = op0;
1214   HOST_WIDE_INT addend_lo, addend_hi;
1215
1216   switch (tls_kind)
1217     {
1218     case TLS_MODEL_GLOBAL_DYNAMIC:
1219       start_sequence ();
1220
1221       tga_op1 = gen_reg_rtx (Pmode);
1222       emit_insn (gen_load_dtpmod (tga_op1, op1));
1223
1224       tga_op2 = gen_reg_rtx (Pmode);
1225       emit_insn (gen_load_dtprel (tga_op2, op1));
1226
1227       tga_ret = emit_library_call_value (gen_tls_get_addr (), NULL_RTX,
1228                                          LCT_CONST, Pmode,
1229                                          tga_op1, Pmode, tga_op2, Pmode);
1230
1231       insns = get_insns ();
1232       end_sequence ();
1233
1234       if (GET_MODE (op0) != Pmode)
1235         op0 = tga_ret;
1236       emit_libcall_block (insns, op0, tga_ret, op1);
1237       break;
1238
1239     case TLS_MODEL_LOCAL_DYNAMIC:
1240       /* ??? This isn't the completely proper way to do local-dynamic
1241          If the call to __tls_get_addr is used only by a single symbol,
1242          then we should (somehow) move the dtprel to the second arg
1243          to avoid the extra add.  */
1244       start_sequence ();
1245
1246       tga_op1 = gen_reg_rtx (Pmode);
1247       emit_insn (gen_load_dtpmod (tga_op1, op1));
1248
1249       tga_op2 = const0_rtx;
1250
1251       tga_ret = emit_library_call_value (gen_tls_get_addr (), NULL_RTX,
1252                                          LCT_CONST, Pmode,
1253                                          tga_op1, Pmode, tga_op2, Pmode);
1254
1255       insns = get_insns ();
1256       end_sequence ();
1257
1258       tga_eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
1259                                 UNSPEC_LD_BASE);
1260       tmp = gen_reg_rtx (Pmode);
1261       emit_libcall_block (insns, tmp, tga_ret, tga_eqv);
1262
1263       if (!register_operand (op0, Pmode))
1264         op0 = gen_reg_rtx (Pmode);
1265       if (TARGET_TLS64)
1266         {
1267           emit_insn (gen_load_dtprel (op0, op1));
1268           emit_insn (gen_adddi3 (op0, tmp, op0));
1269         }
1270       else
1271         emit_insn (gen_add_dtprel (op0, op1, tmp));
1272       break;
1273
1274     case TLS_MODEL_INITIAL_EXEC:
1275       addend_lo = ((addend & 0x3fff) ^ 0x2000) - 0x2000;
1276       addend_hi = addend - addend_lo;
1277
1278       op1 = plus_constant (Pmode, op1, addend_hi);
1279       addend = addend_lo;
1280
1281       tmp = gen_reg_rtx (Pmode);
1282       emit_insn (gen_load_tprel (tmp, op1));
1283
1284       if (!register_operand (op0, Pmode))
1285         op0 = gen_reg_rtx (Pmode);
1286       emit_insn (gen_adddi3 (op0, tmp, gen_thread_pointer ()));
1287       break;
1288
1289     case TLS_MODEL_LOCAL_EXEC:
1290       if (!register_operand (op0, Pmode))
1291         op0 = gen_reg_rtx (Pmode);
1292
1293       op1 = orig_op1;
1294       addend = 0;
1295       if (TARGET_TLS64)
1296         {
1297           emit_insn (gen_load_tprel (op0, op1));
1298           emit_insn (gen_adddi3 (op0, op0, gen_thread_pointer ()));
1299         }
1300       else
1301         emit_insn (gen_add_tprel (op0, op1, gen_thread_pointer ()));
1302       break;
1303
1304     default:
1305       gcc_unreachable ();
1306     }
1307
1308   if (addend)
1309     op0 = expand_simple_binop (Pmode, PLUS, op0, GEN_INT (addend),
1310                                orig_op0, 1, OPTAB_DIRECT);
1311   if (orig_op0 == op0)
1312     return NULL_RTX;
1313   if (GET_MODE (orig_op0) == Pmode)
1314     return op0;
1315   return gen_lowpart (GET_MODE (orig_op0), op0);
1316 }
1317
1318 rtx
1319 ia64_expand_move (rtx op0, rtx op1)
1320 {
1321   machine_mode mode = GET_MODE (op0);
1322
1323   if (!reload_in_progress && !reload_completed && !ia64_move_ok (op0, op1))
1324     op1 = force_reg (mode, op1);
1325
1326   if ((mode == Pmode || mode == ptr_mode) && symbolic_operand (op1, VOIDmode))
1327     {
1328       HOST_WIDE_INT addend = 0;
1329       enum tls_model tls_kind;
1330       rtx sym = op1;
1331
1332       if (GET_CODE (op1) == CONST
1333           && GET_CODE (XEXP (op1, 0)) == PLUS
1334           && GET_CODE (XEXP (XEXP (op1, 0), 1)) == CONST_INT)
1335         {
1336           addend = INTVAL (XEXP (XEXP (op1, 0), 1));
1337           sym = XEXP (XEXP (op1, 0), 0);
1338         }
1339
1340       tls_kind = tls_symbolic_operand_type (sym);
1341       if (tls_kind)
1342         return ia64_expand_tls_address (tls_kind, op0, sym, op1, addend);
1343
1344       if (any_offset_symbol_operand (sym, mode))
1345         addend = 0;
1346       else if (aligned_offset_symbol_operand (sym, mode))
1347         {
1348           HOST_WIDE_INT addend_lo, addend_hi;
1349
1350           addend_lo = ((addend & 0x3fff) ^ 0x2000) - 0x2000;
1351           addend_hi = addend - addend_lo;
1352
1353           if (addend_lo != 0)
1354             {
1355               op1 = plus_constant (mode, sym, addend_hi);
1356               addend = addend_lo;
1357             }
1358           else
1359             addend = 0;
1360         }
1361       else
1362         op1 = sym;
1363
1364       if (reload_completed)
1365         {
1366           /* We really should have taken care of this offset earlier.  */
1367           gcc_assert (addend == 0);
1368           if (ia64_expand_load_address (op0, op1))
1369             return NULL_RTX;
1370         }
1371
1372       if (addend)
1373         {
1374           rtx subtarget = !can_create_pseudo_p () ? op0 : gen_reg_rtx (mode);
1375
1376           emit_insn (gen_rtx_SET (subtarget, op1));
1377
1378           op1 = expand_simple_binop (mode, PLUS, subtarget,
1379                                      GEN_INT (addend), op0, 1, OPTAB_DIRECT);
1380           if (op0 == op1)
1381             return NULL_RTX;
1382         }
1383     }
1384
1385   return op1;
1386 }
1387
1388 /* Split a move from OP1 to OP0 conditional on COND.  */
1389
1390 void
1391 ia64_emit_cond_move (rtx op0, rtx op1, rtx cond)
1392 {
1393   rtx_insn *insn, *first = get_last_insn ();
1394
1395   emit_move_insn (op0, op1);
1396
1397   for (insn = get_last_insn (); insn != first; insn = PREV_INSN (insn))
1398     if (INSN_P (insn))
1399       PATTERN (insn) = gen_rtx_COND_EXEC (VOIDmode, copy_rtx (cond),
1400                                           PATTERN (insn));
1401 }
1402
1403 /* Split a post-reload TImode or TFmode reference into two DImode
1404    components.  This is made extra difficult by the fact that we do
1405    not get any scratch registers to work with, because reload cannot
1406    be prevented from giving us a scratch that overlaps the register
1407    pair involved.  So instead, when addressing memory, we tweak the
1408    pointer register up and back down with POST_INCs.  Or up and not
1409    back down when we can get away with it.
1410
1411    REVERSED is true when the loads must be done in reversed order
1412    (high word first) for correctness.  DEAD is true when the pointer
1413    dies with the second insn we generate and therefore the second
1414    address must not carry a postmodify.
1415
1416    May return an insn which is to be emitted after the moves.  */
1417
1418 static rtx
1419 ia64_split_tmode (rtx out[2], rtx in, bool reversed, bool dead)
1420 {
1421   rtx fixup = 0;
1422
1423   switch (GET_CODE (in))
1424     {
1425     case REG:
1426       out[reversed] = gen_rtx_REG (DImode, REGNO (in));
1427       out[!reversed] = gen_rtx_REG (DImode, REGNO (in) + 1);
1428       break;
1429
1430     case CONST_INT:
1431     case CONST_DOUBLE:
1432       /* Cannot occur reversed.  */
1433       gcc_assert (!reversed);
1434
1435       if (GET_MODE (in) != TFmode)
1436         split_double (in, &out[0], &out[1]);
1437       else
1438         /* split_double does not understand how to split a TFmode
1439            quantity into a pair of DImode constants.  */
1440         {
1441           unsigned HOST_WIDE_INT p[2];
1442           long l[4];  /* TFmode is 128 bits */
1443
1444           real_to_target (l, CONST_DOUBLE_REAL_VALUE (in), TFmode);
1445
1446           if (FLOAT_WORDS_BIG_ENDIAN)
1447             {
1448               p[0] = (((unsigned HOST_WIDE_INT) l[0]) << 32) + l[1];
1449               p[1] = (((unsigned HOST_WIDE_INT) l[2]) << 32) + l[3];
1450             }
1451           else
1452             {
1453               p[0] = (((unsigned HOST_WIDE_INT) l[1]) << 32) + l[0];
1454               p[1] = (((unsigned HOST_WIDE_INT) l[3]) << 32) + l[2];
1455             }
1456           out[0] = GEN_INT (p[0]);
1457           out[1] = GEN_INT (p[1]);
1458         }
1459       break;
1460
1461     case MEM:
1462       {
1463         rtx base = XEXP (in, 0);
1464         rtx offset;
1465
1466         switch (GET_CODE (base))
1467           {
1468           case REG:
1469             if (!reversed)
1470               {
1471                 out[0] = adjust_automodify_address
1472                   (in, DImode, gen_rtx_POST_INC (Pmode, base), 0);
1473                 out[1] = adjust_automodify_address
1474                   (in, DImode, dead ? 0 : gen_rtx_POST_DEC (Pmode, base), 8);
1475               }
1476             else
1477               {
1478                 /* Reversal requires a pre-increment, which can only
1479                    be done as a separate insn.  */
1480                 emit_insn (gen_adddi3 (base, base, GEN_INT (8)));
1481                 out[0] = adjust_automodify_address
1482                   (in, DImode, gen_rtx_POST_DEC (Pmode, base), 8);
1483                 out[1] = adjust_address (in, DImode, 0);
1484               }
1485             break;
1486
1487           case POST_INC:
1488             gcc_assert (!reversed && !dead);
1489
1490             /* Just do the increment in two steps.  */
1491             out[0] = adjust_automodify_address (in, DImode, 0, 0);
1492             out[1] = adjust_automodify_address (in, DImode, 0, 8);
1493             break;
1494
1495           case POST_DEC:
1496             gcc_assert (!reversed && !dead);
1497
1498             /* Add 8, subtract 24.  */
1499             base = XEXP (base, 0);
1500             out[0] = adjust_automodify_address
1501               (in, DImode, gen_rtx_POST_INC (Pmode, base), 0);
1502             out[1] = adjust_automodify_address
1503               (in, DImode,
1504                gen_rtx_POST_MODIFY (Pmode, base,
1505                                     plus_constant (Pmode, base, -24)),
1506                8);
1507             break;
1508
1509           case POST_MODIFY:
1510             gcc_assert (!reversed && !dead);
1511
1512             /* Extract and adjust the modification.  This case is
1513                trickier than the others, because we might have an
1514                index register, or we might have a combined offset that
1515                doesn't fit a signed 9-bit displacement field.  We can
1516                assume the incoming expression is already legitimate.  */
1517             offset = XEXP (base, 1);
1518             base = XEXP (base, 0);
1519
1520             out[0] = adjust_automodify_address
1521               (in, DImode, gen_rtx_POST_INC (Pmode, base), 0);
1522
1523             if (GET_CODE (XEXP (offset, 1)) == REG)
1524               {
1525                 /* Can't adjust the postmodify to match.  Emit the
1526                    original, then a separate addition insn.  */
1527                 out[1] = adjust_automodify_address (in, DImode, 0, 8);
1528                 fixup = gen_adddi3 (base, base, GEN_INT (-8));
1529               }
1530             else
1531               {
1532                 gcc_assert (GET_CODE (XEXP (offset, 1)) == CONST_INT);
1533                 if (INTVAL (XEXP (offset, 1)) < -256 + 8)
1534                   {
1535                     /* Again the postmodify cannot be made to match,
1536                        but in this case it's more efficient to get rid
1537                        of the postmodify entirely and fix up with an
1538                        add insn.  */
1539                     out[1] = adjust_automodify_address (in, DImode, base, 8);
1540                     fixup = gen_adddi3
1541                       (base, base, GEN_INT (INTVAL (XEXP (offset, 1)) - 8));
1542                   }
1543                 else
1544                   {
1545                     /* Combined offset still fits in the displacement field.
1546                        (We cannot overflow it at the high end.)  */
1547                     out[1] = adjust_automodify_address
1548                       (in, DImode, gen_rtx_POST_MODIFY
1549                        (Pmode, base, gen_rtx_PLUS
1550                         (Pmode, base,
1551                          GEN_INT (INTVAL (XEXP (offset, 1)) - 8))),
1552                        8);
1553                   }
1554               }
1555             break;
1556
1557           default:
1558             gcc_unreachable ();
1559           }
1560         break;
1561       }
1562
1563     default:
1564       gcc_unreachable ();
1565     }
1566
1567   return fixup;
1568 }
1569
1570 /* Split a TImode or TFmode move instruction after reload.
1571    This is used by *movtf_internal and *movti_internal.  */
1572 void
1573 ia64_split_tmode_move (rtx operands[])
1574 {
1575   rtx in[2], out[2], insn;
1576   rtx fixup[2];
1577   bool dead = false;
1578   bool reversed = false;
1579
1580   /* It is possible for reload to decide to overwrite a pointer with
1581      the value it points to.  In that case we have to do the loads in
1582      the appropriate order so that the pointer is not destroyed too
1583      early.  Also we must not generate a postmodify for that second
1584      load, or rws_access_regno will die.  And we must not generate a
1585      postmodify for the second load if the destination register
1586      overlaps with the base register.  */
1587   if (GET_CODE (operands[1]) == MEM
1588       && reg_overlap_mentioned_p (operands[0], operands[1]))
1589     {
1590       rtx base = XEXP (operands[1], 0);
1591       while (GET_CODE (base) != REG)
1592         base = XEXP (base, 0);
1593
1594       if (REGNO (base) == REGNO (operands[0]))
1595         reversed = true;
1596
1597       if (refers_to_regno_p (REGNO (operands[0]),
1598                              REGNO (operands[0])+2,
1599                              base, 0))
1600         dead = true;
1601     }
1602   /* Another reason to do the moves in reversed order is if the first
1603      element of the target register pair is also the second element of
1604      the source register pair.  */
1605   if (GET_CODE (operands[0]) == REG && GET_CODE (operands[1]) == REG
1606       && REGNO (operands[0]) == REGNO (operands[1]) + 1)
1607     reversed = true;
1608
1609   fixup[0] = ia64_split_tmode (in, operands[1], reversed, dead);
1610   fixup[1] = ia64_split_tmode (out, operands[0], reversed, dead);
1611
1612 #define MAYBE_ADD_REG_INC_NOTE(INSN, EXP)                               \
1613   if (GET_CODE (EXP) == MEM                                             \
1614       && (GET_CODE (XEXP (EXP, 0)) == POST_MODIFY                       \
1615           || GET_CODE (XEXP (EXP, 0)) == POST_INC                       \
1616           || GET_CODE (XEXP (EXP, 0)) == POST_DEC))                     \
1617     add_reg_note (insn, REG_INC, XEXP (XEXP (EXP, 0), 0))
1618
1619   insn = emit_insn (gen_rtx_SET (out[0], in[0]));
1620   MAYBE_ADD_REG_INC_NOTE (insn, in[0]);
1621   MAYBE_ADD_REG_INC_NOTE (insn, out[0]);
1622
1623   insn = emit_insn (gen_rtx_SET (out[1], in[1]));
1624   MAYBE_ADD_REG_INC_NOTE (insn, in[1]);
1625   MAYBE_ADD_REG_INC_NOTE (insn, out[1]);
1626
1627   if (fixup[0])
1628     emit_insn (fixup[0]);
1629   if (fixup[1])
1630     emit_insn (fixup[1]);
1631
1632 #undef MAYBE_ADD_REG_INC_NOTE
1633 }
1634
1635 /* ??? Fixing GR->FR XFmode moves during reload is hard.  You need to go
1636    through memory plus an extra GR scratch register.  Except that you can
1637    either get the first from TARGET_SECONDARY_MEMORY_NEEDED or the second
1638    from SECONDARY_RELOAD_CLASS, but not both.
1639
1640    We got into problems in the first place by allowing a construct like
1641    (subreg:XF (reg:TI)), which we got from a union containing a long double.
1642    This solution attempts to prevent this situation from occurring.  When
1643    we see something like the above, we spill the inner register to memory.  */
1644
1645 static rtx
1646 spill_xfmode_rfmode_operand (rtx in, int force, machine_mode mode)
1647 {
1648   if (GET_CODE (in) == SUBREG
1649       && GET_MODE (SUBREG_REG (in)) == TImode
1650       && GET_CODE (SUBREG_REG (in)) == REG)
1651     {
1652       rtx memt = assign_stack_temp (TImode, 16);
1653       emit_move_insn (memt, SUBREG_REG (in));
1654       return adjust_address (memt, mode, 0);
1655     }
1656   else if (force && GET_CODE (in) == REG)
1657     {
1658       rtx memx = assign_stack_temp (mode, 16);
1659       emit_move_insn (memx, in);
1660       return memx;
1661     }
1662   else
1663     return in;
1664 }
1665
1666 /* Expand the movxf or movrf pattern (MODE says which) with the given
1667    OPERANDS, returning true if the pattern should then invoke
1668    DONE.  */
1669
1670 bool
1671 ia64_expand_movxf_movrf (machine_mode mode, rtx operands[])
1672 {
1673   rtx op0 = operands[0];
1674
1675   if (GET_CODE (op0) == SUBREG)
1676     op0 = SUBREG_REG (op0);
1677
1678   /* We must support XFmode loads into general registers for stdarg/vararg,
1679      unprototyped calls, and a rare case where a long double is passed as
1680      an argument after a float HFA fills the FP registers.  We split them into
1681      DImode loads for convenience.  We also need to support XFmode stores
1682      for the last case.  This case does not happen for stdarg/vararg routines,
1683      because we do a block store to memory of unnamed arguments.  */
1684
1685   if (GET_CODE (op0) == REG && GR_REGNO_P (REGNO (op0)))
1686     {
1687       rtx out[2];
1688
1689       /* We're hoping to transform everything that deals with XFmode
1690          quantities and GR registers early in the compiler.  */
1691       gcc_assert (can_create_pseudo_p ());
1692
1693       /* Struct to register can just use TImode instead.  */
1694       if ((GET_CODE (operands[1]) == SUBREG
1695            && GET_MODE (SUBREG_REG (operands[1])) == TImode)
1696           || (GET_CODE (operands[1]) == REG
1697               && GR_REGNO_P (REGNO (operands[1]))))
1698         {
1699           rtx op1 = operands[1];
1700
1701           if (GET_CODE (op1) == SUBREG)
1702             op1 = SUBREG_REG (op1);
1703           else
1704             op1 = gen_rtx_REG (TImode, REGNO (op1));
1705
1706           emit_move_insn (gen_rtx_REG (TImode, REGNO (op0)), op1);
1707           return true;
1708         }
1709
1710       if (GET_CODE (operands[1]) == CONST_DOUBLE)
1711         {
1712           /* Don't word-swap when reading in the constant.  */
1713           emit_move_insn (gen_rtx_REG (DImode, REGNO (op0)),
1714                           operand_subword (operands[1], WORDS_BIG_ENDIAN,
1715                                            0, mode));
1716           emit_move_insn (gen_rtx_REG (DImode, REGNO (op0) + 1),
1717                           operand_subword (operands[1], !WORDS_BIG_ENDIAN,
1718                                            0, mode));
1719           return true;
1720         }
1721
1722       /* If the quantity is in a register not known to be GR, spill it.  */
1723       if (register_operand (operands[1], mode))
1724         operands[1] = spill_xfmode_rfmode_operand (operands[1], 1, mode);
1725
1726       gcc_assert (GET_CODE (operands[1]) == MEM);
1727
1728       /* Don't word-swap when reading in the value.  */
1729       out[0] = gen_rtx_REG (DImode, REGNO (op0));
1730       out[1] = gen_rtx_REG (DImode, REGNO (op0) + 1);
1731
1732       emit_move_insn (out[0], adjust_address (operands[1], DImode, 0));
1733       emit_move_insn (out[1], adjust_address (operands[1], DImode, 8));
1734       return true;
1735     }
1736
1737   if (GET_CODE (operands[1]) == REG && GR_REGNO_P (REGNO (operands[1])))
1738     {
1739       /* We're hoping to transform everything that deals with XFmode
1740          quantities and GR registers early in the compiler.  */
1741       gcc_assert (can_create_pseudo_p ());
1742
1743       /* Op0 can't be a GR_REG here, as that case is handled above.
1744          If op0 is a register, then we spill op1, so that we now have a
1745          MEM operand.  This requires creating an XFmode subreg of a TImode reg
1746          to force the spill.  */
1747       if (register_operand (operands[0], mode))
1748         {
1749           rtx op1 = gen_rtx_REG (TImode, REGNO (operands[1]));
1750           op1 = gen_rtx_SUBREG (mode, op1, 0);
1751           operands[1] = spill_xfmode_rfmode_operand (op1, 0, mode);
1752         }
1753
1754       else
1755         {
1756           rtx in[2];
1757
1758           gcc_assert (GET_CODE (operands[0]) == MEM);
1759
1760           /* Don't word-swap when writing out the value.  */
1761           in[0] = gen_rtx_REG (DImode, REGNO (operands[1]));
1762           in[1] = gen_rtx_REG (DImode, REGNO (operands[1]) + 1);
1763
1764           emit_move_insn (adjust_address (operands[0], DImode, 0), in[0]);
1765           emit_move_insn (adjust_address (operands[0], DImode, 8), in[1]);
1766           return true;
1767         }
1768     }
1769
1770   if (!reload_in_progress && !reload_completed)
1771     {
1772       operands[1] = spill_xfmode_rfmode_operand (operands[1], 0, mode);
1773
1774       if (GET_MODE (op0) == TImode && GET_CODE (op0) == REG)
1775         {
1776           rtx memt, memx, in = operands[1];
1777           if (CONSTANT_P (in))
1778             in = validize_mem (force_const_mem (mode, in));
1779           if (GET_CODE (in) == MEM)
1780             memt = adjust_address (in, TImode, 0);
1781           else
1782             {
1783               memt = assign_stack_temp (TImode, 16);
1784               memx = adjust_address (memt, mode, 0);
1785               emit_move_insn (memx, in);
1786             }
1787           emit_move_insn (op0, memt);
1788           return true;
1789         }
1790
1791       if (!ia64_move_ok (operands[0], operands[1]))
1792         operands[1] = force_reg (mode, operands[1]);
1793     }
1794
1795   return false;
1796 }
1797
1798 /* Emit comparison instruction if necessary, replacing *EXPR, *OP0, *OP1
1799    with the expression that holds the compare result (in VOIDmode).  */
1800
1801 static GTY(()) rtx cmptf_libfunc;
1802
1803 void
1804 ia64_expand_compare (rtx *expr, rtx *op0, rtx *op1)
1805 {
1806   enum rtx_code code = GET_CODE (*expr);
1807   rtx cmp;
1808
1809   /* If we have a BImode input, then we already have a compare result, and
1810      do not need to emit another comparison.  */
1811   if (GET_MODE (*op0) == BImode)
1812     {
1813       gcc_assert ((code == NE || code == EQ) && *op1 == const0_rtx);
1814       cmp = *op0;
1815     }
1816   /* HPUX TFmode compare requires a library call to _U_Qfcmp, which takes a
1817      magic number as its third argument, that indicates what to do.
1818      The return value is an integer to be compared against zero.  */
1819   else if (TARGET_HPUX && GET_MODE (*op0) == TFmode)
1820     {
1821       enum qfcmp_magic {
1822         QCMP_INV = 1,   /* Raise FP_INVALID on NaNs as a side effect.  */
1823         QCMP_UNORD = 2,
1824         QCMP_EQ = 4,
1825         QCMP_LT = 8,
1826         QCMP_GT = 16
1827       };
1828       int magic;
1829       enum rtx_code ncode;
1830       rtx ret;
1831
1832       gcc_assert (cmptf_libfunc && GET_MODE (*op1) == TFmode);
1833       switch (code)
1834         {
1835           /* 1 = equal, 0 = not equal.  Equality operators do
1836              not raise FP_INVALID when given a NaN operand.  */
1837         case EQ:        magic = QCMP_EQ;                  ncode = NE; break;
1838         case NE:        magic = QCMP_EQ;                  ncode = EQ; break;
1839           /* isunordered() from C99.  */
1840         case UNORDERED: magic = QCMP_UNORD;               ncode = NE; break;
1841         case ORDERED:   magic = QCMP_UNORD;               ncode = EQ; break;
1842           /* Relational operators raise FP_INVALID when given
1843              a NaN operand.  */
1844         case LT:        magic = QCMP_LT        |QCMP_INV; ncode = NE; break;
1845         case LE:        magic = QCMP_LT|QCMP_EQ|QCMP_INV; ncode = NE; break;
1846         case GT:        magic = QCMP_GT        |QCMP_INV; ncode = NE; break;
1847         case GE:        magic = QCMP_GT|QCMP_EQ|QCMP_INV; ncode = NE; break;
1848           /* Unordered relational operators do not raise FP_INVALID
1849              when given a NaN operand.  */
1850         case UNLT:    magic = QCMP_LT        |QCMP_UNORD; ncode = NE; break;
1851         case UNLE:    magic = QCMP_LT|QCMP_EQ|QCMP_UNORD; ncode = NE; break;
1852         case UNGT:    magic = QCMP_GT        |QCMP_UNORD; ncode = NE; break;
1853         case UNGE:    magic = QCMP_GT|QCMP_EQ|QCMP_UNORD; ncode = NE; break;
1854           /* Not supported.  */
1855         case UNEQ:
1856         case LTGT:
1857         default: gcc_unreachable ();
1858         }
1859
1860       start_sequence ();
1861
1862       ret = emit_library_call_value (cmptf_libfunc, 0, LCT_CONST, DImode,
1863                                      *op0, TFmode, *op1, TFmode,
1864                                      GEN_INT (magic), DImode);
1865       cmp = gen_reg_rtx (BImode);
1866       emit_insn (gen_rtx_SET (cmp, gen_rtx_fmt_ee (ncode, BImode,
1867                                                    ret, const0_rtx)));
1868
1869       rtx_insn *insns = get_insns ();
1870       end_sequence ();
1871
1872       emit_libcall_block (insns, cmp, cmp,
1873                           gen_rtx_fmt_ee (code, BImode, *op0, *op1));
1874       code = NE;
1875     }
1876   else
1877     {
1878       cmp = gen_reg_rtx (BImode);
1879       emit_insn (gen_rtx_SET (cmp, gen_rtx_fmt_ee (code, BImode, *op0, *op1)));
1880       code = NE;
1881     }
1882
1883   *expr = gen_rtx_fmt_ee (code, VOIDmode, cmp, const0_rtx);
1884   *op0 = cmp;
1885   *op1 = const0_rtx;
1886 }
1887
1888 /* Generate an integral vector comparison.  Return true if the condition has
1889    been reversed, and so the sense of the comparison should be inverted.  */
1890
1891 static bool
1892 ia64_expand_vecint_compare (enum rtx_code code, machine_mode mode,
1893                             rtx dest, rtx op0, rtx op1)
1894 {
1895   bool negate = false;
1896   rtx x;
1897
1898   /* Canonicalize the comparison to EQ, GT, GTU.  */
1899   switch (code)
1900     {
1901     case EQ:
1902     case GT:
1903     case GTU:
1904       break;
1905
1906     case NE:
1907     case LE:
1908     case LEU:
1909       code = reverse_condition (code);
1910       negate = true;
1911       break;
1912
1913     case GE:
1914     case GEU:
1915       code = reverse_condition (code);
1916       negate = true;
1917       /* FALLTHRU */
1918
1919     case LT:
1920     case LTU:
1921       code = swap_condition (code);
1922       x = op0, op0 = op1, op1 = x;
1923       break;
1924
1925     default:
1926       gcc_unreachable ();
1927     }
1928
1929   /* Unsigned parallel compare is not supported by the hardware.  Play some
1930      tricks to turn this into a signed comparison against 0.  */
1931   if (code == GTU)
1932     {
1933       switch (mode)
1934         {
1935         case E_V2SImode:
1936           {
1937             rtx t1, t2, mask;
1938
1939             /* Subtract (-(INT MAX) - 1) from both operands to make
1940                them signed.  */
1941             mask = gen_int_mode (0x80000000, SImode);
1942             mask = gen_const_vec_duplicate (V2SImode, mask);
1943             mask = force_reg (mode, mask);
1944             t1 = gen_reg_rtx (mode);
1945             emit_insn (gen_subv2si3 (t1, op0, mask));
1946             t2 = gen_reg_rtx (mode);
1947             emit_insn (gen_subv2si3 (t2, op1, mask));
1948             op0 = t1;
1949             op1 = t2;
1950             code = GT;
1951           }
1952           break;
1953
1954         case E_V8QImode:
1955         case E_V4HImode:
1956           /* Perform a parallel unsigned saturating subtraction.  */
1957           x = gen_reg_rtx (mode);
1958           emit_insn (gen_rtx_SET (x, gen_rtx_US_MINUS (mode, op0, op1)));
1959
1960           code = EQ;
1961           op0 = x;
1962           op1 = CONST0_RTX (mode);
1963           negate = !negate;
1964           break;
1965
1966         default:
1967           gcc_unreachable ();
1968         }
1969     }
1970
1971   x = gen_rtx_fmt_ee (code, mode, op0, op1);
1972   emit_insn (gen_rtx_SET (dest, x));
1973
1974   return negate;
1975 }
1976
1977 /* Emit an integral vector conditional move.  */
1978
1979 void
1980 ia64_expand_vecint_cmov (rtx operands[])
1981 {
1982   machine_mode mode = GET_MODE (operands[0]);
1983   enum rtx_code code = GET_CODE (operands[3]);
1984   bool negate;
1985   rtx cmp, x, ot, of;
1986
1987   cmp = gen_reg_rtx (mode);
1988   negate = ia64_expand_vecint_compare (code, mode, cmp,
1989                                        operands[4], operands[5]);
1990
1991   ot = operands[1+negate];
1992   of = operands[2-negate];
1993
1994   if (ot == CONST0_RTX (mode))
1995     {
1996       if (of == CONST0_RTX (mode))
1997         {
1998           emit_move_insn (operands[0], ot);
1999           return;
2000         }
2001
2002       x = gen_rtx_NOT (mode, cmp);
2003       x = gen_rtx_AND (mode, x, of);
2004       emit_insn (gen_rtx_SET (operands[0], x));
2005     }
2006   else if (of == CONST0_RTX (mode))
2007     {
2008       x = gen_rtx_AND (mode, cmp, ot);
2009       emit_insn (gen_rtx_SET (operands[0], x));
2010     }
2011   else
2012     {
2013       rtx t, f;
2014
2015       t = gen_reg_rtx (mode);
2016       x = gen_rtx_AND (mode, cmp, operands[1+negate]);
2017       emit_insn (gen_rtx_SET (t, x));
2018
2019       f = gen_reg_rtx (mode);
2020       x = gen_rtx_NOT (mode, cmp);
2021       x = gen_rtx_AND (mode, x, operands[2-negate]);
2022       emit_insn (gen_rtx_SET (f, x));
2023
2024       x = gen_rtx_IOR (mode, t, f);
2025       emit_insn (gen_rtx_SET (operands[0], x));
2026     }
2027 }
2028
2029 /* Emit an integral vector min or max operation.  Return true if all done.  */
2030
2031 bool
2032 ia64_expand_vecint_minmax (enum rtx_code code, machine_mode mode,
2033                            rtx operands[])
2034 {
2035   rtx xops[6];
2036
2037   /* These four combinations are supported directly.  */
2038   if (mode == V8QImode && (code == UMIN || code == UMAX))
2039     return false;
2040   if (mode == V4HImode && (code == SMIN || code == SMAX))
2041     return false;
2042
2043   /* This combination can be implemented with only saturating subtraction.  */
2044   if (mode == V4HImode && code == UMAX)
2045     {
2046       rtx x, tmp = gen_reg_rtx (mode);
2047
2048       x = gen_rtx_US_MINUS (mode, operands[1], operands[2]);
2049       emit_insn (gen_rtx_SET (tmp, x));
2050
2051       emit_insn (gen_addv4hi3 (operands[0], tmp, operands[2]));
2052       return true;
2053     }
2054
2055   /* Everything else implemented via vector comparisons.  */
2056   xops[0] = operands[0];
2057   xops[4] = xops[1] = operands[1];
2058   xops[5] = xops[2] = operands[2];
2059
2060   switch (code)
2061     {
2062     case UMIN:
2063       code = LTU;
2064       break;
2065     case UMAX:
2066       code = GTU;
2067       break;
2068     case SMIN:
2069       code = LT;
2070       break;
2071     case SMAX:
2072       code = GT;
2073       break;
2074     default:
2075       gcc_unreachable ();
2076     }
2077   xops[3] = gen_rtx_fmt_ee (code, VOIDmode, operands[1], operands[2]);
2078
2079   ia64_expand_vecint_cmov (xops);
2080   return true;
2081 }
2082
2083 /* The vectors LO and HI each contain N halves of a double-wide vector.
2084    Reassemble either the first N/2 or the second N/2 elements.  */
2085
2086 void
2087 ia64_unpack_assemble (rtx out, rtx lo, rtx hi, bool highp)
2088 {
2089   machine_mode vmode = GET_MODE (lo);
2090   unsigned int i, high, nelt = GET_MODE_NUNITS (vmode);
2091   struct expand_vec_perm_d d;
2092   bool ok;
2093
2094   d.target = gen_lowpart (vmode, out);
2095   d.op0 = (TARGET_BIG_ENDIAN ? hi : lo);
2096   d.op1 = (TARGET_BIG_ENDIAN ? lo : hi);
2097   d.vmode = vmode;
2098   d.nelt = nelt;
2099   d.one_operand_p = false;
2100   d.testing_p = false;
2101
2102   high = (highp ? nelt / 2 : 0);
2103   for (i = 0; i < nelt / 2; ++i)
2104     {
2105       d.perm[i * 2] = i + high;
2106       d.perm[i * 2 + 1] = i + high + nelt;
2107     }
2108
2109   ok = ia64_expand_vec_perm_const_1 (&d);
2110   gcc_assert (ok);
2111 }
2112
2113 /* Return a vector of the sign-extension of VEC.  */
2114
2115 static rtx
2116 ia64_unpack_sign (rtx vec, bool unsignedp)
2117 {
2118   machine_mode mode = GET_MODE (vec);
2119   rtx zero = CONST0_RTX (mode);
2120
2121   if (unsignedp)
2122     return zero;
2123   else
2124     {
2125       rtx sign = gen_reg_rtx (mode);
2126       bool neg;
2127
2128       neg = ia64_expand_vecint_compare (LT, mode, sign, vec, zero);
2129       gcc_assert (!neg);
2130
2131       return sign;
2132     }
2133 }
2134
2135 /* Emit an integral vector unpack operation.  */
2136
2137 void
2138 ia64_expand_unpack (rtx operands[3], bool unsignedp, bool highp)
2139 {
2140   rtx sign = ia64_unpack_sign (operands[1], unsignedp);
2141   ia64_unpack_assemble (operands[0], operands[1], sign, highp);
2142 }
2143
2144 /* Emit an integral vector widening sum operations.  */
2145
2146 void
2147 ia64_expand_widen_sum (rtx operands[3], bool unsignedp)
2148 {
2149   machine_mode wmode;
2150   rtx l, h, t, sign;
2151
2152   sign = ia64_unpack_sign (operands[1], unsignedp);
2153
2154   wmode = GET_MODE (operands[0]);
2155   l = gen_reg_rtx (wmode);
2156   h = gen_reg_rtx (wmode);
2157
2158   ia64_unpack_assemble (l, operands[1], sign, false);
2159   ia64_unpack_assemble (h, operands[1], sign, true);
2160
2161   t = expand_binop (wmode, add_optab, l, operands[2], NULL, 0, OPTAB_DIRECT);
2162   t = expand_binop (wmode, add_optab, h, t, operands[0], 0, OPTAB_DIRECT);
2163   if (t != operands[0])
2164     emit_move_insn (operands[0], t);
2165 }
2166
2167 /* Emit the appropriate sequence for a call.  */
2168
2169 void
2170 ia64_expand_call (rtx retval, rtx addr, rtx nextarg ATTRIBUTE_UNUSED,
2171                   int sibcall_p)
2172 {
2173   rtx insn, b0;
2174
2175   addr = XEXP (addr, 0);
2176   addr = convert_memory_address (DImode, addr);
2177   b0 = gen_rtx_REG (DImode, R_BR (0));
2178
2179   /* ??? Should do this for functions known to bind local too.  */
2180   if (TARGET_NO_PIC || TARGET_AUTO_PIC)
2181     {
2182       if (sibcall_p)
2183         insn = gen_sibcall_nogp (addr);
2184       else if (! retval)
2185         insn = gen_call_nogp (addr, b0);
2186       else
2187         insn = gen_call_value_nogp (retval, addr, b0);
2188       insn = emit_call_insn (insn);
2189     }
2190   else
2191     {
2192       if (sibcall_p)
2193         insn = gen_sibcall_gp (addr);
2194       else if (! retval)
2195         insn = gen_call_gp (addr, b0);
2196       else
2197         insn = gen_call_value_gp (retval, addr, b0);
2198       insn = emit_call_insn (insn);
2199
2200       use_reg (&CALL_INSN_FUNCTION_USAGE (insn), pic_offset_table_rtx);
2201     }
2202
2203   if (sibcall_p)
2204     use_reg (&CALL_INSN_FUNCTION_USAGE (insn), b0);
2205
2206   if (TARGET_ABI_OPEN_VMS)
2207     use_reg (&CALL_INSN_FUNCTION_USAGE (insn),
2208              gen_rtx_REG (DImode, GR_REG (25)));
2209 }
2210
2211 static void
2212 reg_emitted (enum ia64_frame_regs r)
2213 {
2214   if (emitted_frame_related_regs[r] == 0)
2215     emitted_frame_related_regs[r] = current_frame_info.r[r];
2216   else
2217     gcc_assert (emitted_frame_related_regs[r] == current_frame_info.r[r]);
2218 }
2219
2220 static int
2221 get_reg (enum ia64_frame_regs r)
2222 {
2223   reg_emitted (r);
2224   return current_frame_info.r[r];
2225 }
2226
2227 static bool
2228 is_emitted (int regno)
2229 {
2230   unsigned int r;
2231
2232   for (r = reg_fp; r < number_of_ia64_frame_regs; r++)
2233     if (emitted_frame_related_regs[r] == regno)
2234       return true;
2235   return false;
2236 }
2237
2238 void
2239 ia64_reload_gp (void)
2240 {
2241   rtx tmp;
2242
2243   if (current_frame_info.r[reg_save_gp])
2244     {
2245       tmp = gen_rtx_REG (DImode, get_reg (reg_save_gp));
2246     }
2247   else
2248     {
2249       HOST_WIDE_INT offset;
2250       rtx offset_r;
2251
2252       offset = (current_frame_info.spill_cfa_off
2253                 + current_frame_info.spill_size);
2254       if (frame_pointer_needed)
2255         {
2256           tmp = hard_frame_pointer_rtx;
2257           offset = -offset;
2258         }
2259       else
2260         {
2261           tmp = stack_pointer_rtx;
2262           offset = current_frame_info.total_size - offset;
2263         }
2264
2265       offset_r = GEN_INT (offset);
2266       if (satisfies_constraint_I (offset_r))
2267         emit_insn (gen_adddi3 (pic_offset_table_rtx, tmp, offset_r));
2268       else
2269         {
2270           emit_move_insn (pic_offset_table_rtx, offset_r);
2271           emit_insn (gen_adddi3 (pic_offset_table_rtx,
2272                                  pic_offset_table_rtx, tmp));
2273         }
2274
2275       tmp = gen_rtx_MEM (DImode, pic_offset_table_rtx);
2276     }
2277
2278   emit_move_insn (pic_offset_table_rtx, tmp);
2279 }
2280
2281 void
2282 ia64_split_call (rtx retval, rtx addr, rtx retaddr, rtx scratch_r,
2283                  rtx scratch_b, int noreturn_p, int sibcall_p)
2284 {
2285   rtx insn;
2286   bool is_desc = false;
2287
2288   /* If we find we're calling through a register, then we're actually
2289      calling through a descriptor, so load up the values.  */
2290   if (REG_P (addr) && GR_REGNO_P (REGNO (addr)))
2291     {
2292       rtx tmp;
2293       bool addr_dead_p;
2294
2295       /* ??? We are currently constrained to *not* use peep2, because
2296          we can legitimately change the global lifetime of the GP
2297          (in the form of killing where previously live).  This is
2298          because a call through a descriptor doesn't use the previous
2299          value of the GP, while a direct call does, and we do not
2300          commit to either form until the split here.
2301
2302          That said, this means that we lack precise life info for
2303          whether ADDR is dead after this call.  This is not terribly
2304          important, since we can fix things up essentially for free
2305          with the POST_DEC below, but it's nice to not use it when we
2306          can immediately tell it's not necessary.  */
2307       addr_dead_p = ((noreturn_p || sibcall_p
2308                       || TEST_HARD_REG_BIT (regs_invalidated_by_call,
2309                                             REGNO (addr)))
2310                      && !FUNCTION_ARG_REGNO_P (REGNO (addr)));
2311
2312       /* Load the code address into scratch_b.  */
2313       tmp = gen_rtx_POST_INC (Pmode, addr);
2314       tmp = gen_rtx_MEM (Pmode, tmp);
2315       emit_move_insn (scratch_r, tmp);
2316       emit_move_insn (scratch_b, scratch_r);
2317
2318       /* Load the GP address.  If ADDR is not dead here, then we must
2319          revert the change made above via the POST_INCREMENT.  */
2320       if (!addr_dead_p)
2321         tmp = gen_rtx_POST_DEC (Pmode, addr);
2322       else
2323         tmp = addr;
2324       tmp = gen_rtx_MEM (Pmode, tmp);
2325       emit_move_insn (pic_offset_table_rtx, tmp);
2326
2327       is_desc = true;
2328       addr = scratch_b;
2329     }
2330
2331   if (sibcall_p)
2332     insn = gen_sibcall_nogp (addr);
2333   else if (retval)
2334     insn = gen_call_value_nogp (retval, addr, retaddr);
2335   else
2336     insn = gen_call_nogp (addr, retaddr);
2337   emit_call_insn (insn);
2338
2339   if ((!TARGET_CONST_GP || is_desc) && !noreturn_p && !sibcall_p)
2340     ia64_reload_gp ();
2341 }
2342
2343 /* Expand an atomic operation.  We want to perform MEM <CODE>= VAL atomically.
2344
2345    This differs from the generic code in that we know about the zero-extending
2346    properties of cmpxchg, and the zero-extending requirements of ar.ccv.  We
2347    also know that ld.acq+cmpxchg.rel equals a full barrier.
2348
2349    The loop we want to generate looks like
2350
2351         cmp_reg = mem;
2352       label:
2353         old_reg = cmp_reg;
2354         new_reg = cmp_reg op val;
2355         cmp_reg = compare-and-swap(mem, old_reg, new_reg)
2356         if (cmp_reg != old_reg)
2357           goto label;
2358
2359    Note that we only do the plain load from memory once.  Subsequent
2360    iterations use the value loaded by the compare-and-swap pattern.  */
2361
2362 void
2363 ia64_expand_atomic_op (enum rtx_code code, rtx mem, rtx val,
2364                        rtx old_dst, rtx new_dst, enum memmodel model)
2365 {
2366   machine_mode mode = GET_MODE (mem);
2367   rtx old_reg, new_reg, cmp_reg, ar_ccv, label;
2368   enum insn_code icode;
2369
2370   /* Special case for using fetchadd.  */
2371   if ((mode == SImode || mode == DImode)
2372       && (code == PLUS || code == MINUS)
2373       && fetchadd_operand (val, mode))
2374     {
2375       if (code == MINUS)
2376         val = GEN_INT (-INTVAL (val));
2377
2378       if (!old_dst)
2379         old_dst = gen_reg_rtx (mode);
2380
2381       switch (model)
2382         {
2383         case MEMMODEL_ACQ_REL:
2384         case MEMMODEL_SEQ_CST:
2385         case MEMMODEL_SYNC_SEQ_CST:
2386           emit_insn (gen_memory_barrier ());
2387           /* FALLTHRU */
2388         case MEMMODEL_RELAXED:
2389         case MEMMODEL_ACQUIRE:
2390         case MEMMODEL_SYNC_ACQUIRE:
2391         case MEMMODEL_CONSUME:
2392           if (mode == SImode)
2393             icode = CODE_FOR_fetchadd_acq_si;
2394           else
2395             icode = CODE_FOR_fetchadd_acq_di;
2396           break;
2397         case MEMMODEL_RELEASE:
2398         case MEMMODEL_SYNC_RELEASE:
2399           if (mode == SImode)
2400             icode = CODE_FOR_fetchadd_rel_si;
2401           else
2402             icode = CODE_FOR_fetchadd_rel_di;
2403           break;
2404
2405         default:
2406           gcc_unreachable ();
2407         }
2408
2409       emit_insn (GEN_FCN (icode) (old_dst, mem, val));
2410
2411       if (new_dst)
2412         {
2413           new_reg = expand_simple_binop (mode, PLUS, old_dst, val, new_dst,
2414                                          true, OPTAB_WIDEN);
2415           if (new_reg != new_dst)
2416             emit_move_insn (new_dst, new_reg);
2417         }
2418       return;
2419     }
2420
2421   /* Because of the volatile mem read, we get an ld.acq, which is the
2422      front half of the full barrier.  The end half is the cmpxchg.rel.
2423      For relaxed and release memory models, we don't need this.  But we
2424      also don't bother trying to prevent it either.  */
2425   gcc_assert (is_mm_relaxed (model) || is_mm_release (model)
2426               || MEM_VOLATILE_P (mem));
2427
2428   old_reg = gen_reg_rtx (DImode);
2429   cmp_reg = gen_reg_rtx (DImode);
2430   label = gen_label_rtx ();
2431
2432   if (mode != DImode)
2433     {
2434       val = simplify_gen_subreg (DImode, val, mode, 0);
2435       emit_insn (gen_extend_insn (cmp_reg, mem, DImode, mode, 1));
2436     }
2437   else
2438     emit_move_insn (cmp_reg, mem);
2439
2440   emit_label (label);
2441
2442   ar_ccv = gen_rtx_REG (DImode, AR_CCV_REGNUM);
2443   emit_move_insn (old_reg, cmp_reg);
2444   emit_move_insn (ar_ccv, cmp_reg);
2445
2446   if (old_dst)
2447     emit_move_insn (old_dst, gen_lowpart (mode, cmp_reg));
2448
2449   new_reg = cmp_reg;
2450   if (code == NOT)
2451     {
2452       new_reg = expand_simple_binop (DImode, AND, new_reg, val, NULL_RTX,
2453                                      true, OPTAB_DIRECT);
2454       new_reg = expand_simple_unop (DImode, code, new_reg, NULL_RTX, true);
2455     }
2456   else
2457     new_reg = expand_simple_binop (DImode, code, new_reg, val, NULL_RTX,
2458                                    true, OPTAB_DIRECT);
2459
2460   if (mode != DImode)
2461     new_reg = gen_lowpart (mode, new_reg);
2462   if (new_dst)
2463     emit_move_insn (new_dst, new_reg);
2464
2465   switch (model)
2466     {
2467     case MEMMODEL_RELAXED:
2468     case MEMMODEL_ACQUIRE:
2469     case MEMMODEL_SYNC_ACQUIRE:
2470     case MEMMODEL_CONSUME:
2471       switch (mode)
2472         {
2473         case E_QImode: icode = CODE_FOR_cmpxchg_acq_qi;  break;
2474         case E_HImode: icode = CODE_FOR_cmpxchg_acq_hi;  break;
2475         case E_SImode: icode = CODE_FOR_cmpxchg_acq_si;  break;
2476         case E_DImode: icode = CODE_FOR_cmpxchg_acq_di;  break;
2477         default:
2478           gcc_unreachable ();
2479         }
2480       break;
2481
2482     case MEMMODEL_RELEASE:
2483     case MEMMODEL_SYNC_RELEASE:
2484     case MEMMODEL_ACQ_REL:
2485     case MEMMODEL_SEQ_CST:
2486     case MEMMODEL_SYNC_SEQ_CST:
2487       switch (mode)
2488         {
2489         case E_QImode: icode = CODE_FOR_cmpxchg_rel_qi;  break;
2490         case E_HImode: icode = CODE_FOR_cmpxchg_rel_hi;  break;
2491         case E_SImode: icode = CODE_FOR_cmpxchg_rel_si;  break;
2492         case E_DImode: icode = CODE_FOR_cmpxchg_rel_di;  break;
2493         default:
2494           gcc_unreachable ();
2495         }
2496       break;
2497
2498     default:
2499       gcc_unreachable ();
2500     }
2501
2502   emit_insn (GEN_FCN (icode) (cmp_reg, mem, ar_ccv, new_reg));
2503
2504   emit_cmp_and_jump_insns (cmp_reg, old_reg, NE, NULL, DImode, true, label);
2505 }
2506 \f
2507 /* Begin the assembly file.  */
2508
2509 static void
2510 ia64_file_start (void)
2511 {
2512   default_file_start ();
2513   emit_safe_across_calls ();
2514 }
2515
2516 void
2517 emit_safe_across_calls (void)
2518 {
2519   unsigned int rs, re;
2520   int out_state;
2521
2522   rs = 1;
2523   out_state = 0;
2524   while (1)
2525     {
2526       while (rs < 64 && call_used_regs[PR_REG (rs)])
2527         rs++;
2528       if (rs >= 64)
2529         break;
2530       for (re = rs + 1; re < 64 && ! call_used_regs[PR_REG (re)]; re++)
2531         continue;
2532       if (out_state == 0)
2533         {
2534           fputs ("\t.pred.safe_across_calls ", asm_out_file);
2535           out_state = 1;
2536         }
2537       else
2538         fputc (',', asm_out_file);
2539       if (re == rs + 1)
2540         fprintf (asm_out_file, "p%u", rs);
2541       else
2542         fprintf (asm_out_file, "p%u-p%u", rs, re - 1);
2543       rs = re + 1;
2544     }
2545   if (out_state)
2546     fputc ('\n', asm_out_file);
2547 }
2548
2549 /* Globalize a declaration.  */
2550
2551 static void
2552 ia64_globalize_decl_name (FILE * stream, tree decl)
2553 {
2554   const char *name = XSTR (XEXP (DECL_RTL (decl), 0), 0);
2555   tree version_attr = lookup_attribute ("version_id", DECL_ATTRIBUTES (decl));
2556   if (version_attr)
2557     {
2558       tree v = TREE_VALUE (TREE_VALUE (version_attr));
2559       const char *p = TREE_STRING_POINTER (v);
2560       fprintf (stream, "\t.alias %s#, \"%s{%s}\"\n", name, name, p);
2561     }
2562   targetm.asm_out.globalize_label (stream, name);
2563   if (TREE_CODE (decl) == FUNCTION_DECL)
2564     ASM_OUTPUT_TYPE_DIRECTIVE (stream, name, "function");
2565 }
2566
2567 /* Helper function for ia64_compute_frame_size: find an appropriate general
2568    register to spill some special register to.  SPECIAL_SPILL_MASK contains
2569    bits in GR0 to GR31 that have already been allocated by this routine.
2570    TRY_LOCALS is true if we should attempt to locate a local regnum.  */
2571
2572 static int
2573 find_gr_spill (enum ia64_frame_regs r, int try_locals)
2574 {
2575   int regno;
2576
2577   if (emitted_frame_related_regs[r] != 0)
2578     {
2579       regno = emitted_frame_related_regs[r];
2580       if (regno >= LOC_REG (0) && regno < LOC_REG (80 - frame_pointer_needed)
2581           && current_frame_info.n_local_regs < regno - LOC_REG (0) + 1)
2582         current_frame_info.n_local_regs = regno - LOC_REG (0) + 1;
2583       else if (crtl->is_leaf
2584                && regno >= GR_REG (1) && regno <= GR_REG (31))
2585         current_frame_info.gr_used_mask |= 1 << regno;
2586
2587       return regno;
2588     }
2589
2590   /* If this is a leaf function, first try an otherwise unused
2591      call-clobbered register.  */
2592   if (crtl->is_leaf)
2593     {
2594       for (regno = GR_REG (1); regno <= GR_REG (31); regno++)
2595         if (! df_regs_ever_live_p (regno)
2596             && call_used_regs[regno]
2597             && ! fixed_regs[regno]
2598             && ! global_regs[regno]
2599             && ((current_frame_info.gr_used_mask >> regno) & 1) == 0
2600             && ! is_emitted (regno))
2601           {
2602             current_frame_info.gr_used_mask |= 1 << regno;
2603             return regno;
2604           }
2605     }
2606
2607   if (try_locals)
2608     {
2609       regno = current_frame_info.n_local_regs;
2610       /* If there is a frame pointer, then we can't use loc79, because
2611          that is HARD_FRAME_POINTER_REGNUM.  In particular, see the
2612          reg_name switching code in ia64_expand_prologue.  */
2613       while (regno < (80 - frame_pointer_needed))
2614         if (! is_emitted (LOC_REG (regno++)))
2615           {
2616             current_frame_info.n_local_regs = regno;
2617             return LOC_REG (regno - 1);
2618           }
2619     }
2620
2621   /* Failed to find a general register to spill to.  Must use stack.  */
2622   return 0;
2623 }
2624
2625 /* In order to make for nice schedules, we try to allocate every temporary
2626    to a different register.  We must of course stay away from call-saved,
2627    fixed, and global registers.  We must also stay away from registers
2628    allocated in current_frame_info.gr_used_mask, since those include regs
2629    used all through the prologue.
2630
2631    Any register allocated here must be used immediately.  The idea is to
2632    aid scheduling, not to solve data flow problems.  */
2633
2634 static int last_scratch_gr_reg;
2635
2636 static int
2637 next_scratch_gr_reg (void)
2638 {
2639   int i, regno;
2640
2641   for (i = 0; i < 32; ++i)
2642     {
2643       regno = (last_scratch_gr_reg + i + 1) & 31;
2644       if (call_used_regs[regno]
2645           && ! fixed_regs[regno]
2646           && ! global_regs[regno]
2647           && ((current_frame_info.gr_used_mask >> regno) & 1) == 0)
2648         {
2649           last_scratch_gr_reg = regno;
2650           return regno;
2651         }
2652     }
2653
2654   /* There must be _something_ available.  */
2655   gcc_unreachable ();
2656 }
2657
2658 /* Helper function for ia64_compute_frame_size, called through
2659    diddle_return_value.  Mark REG in current_frame_info.gr_used_mask.  */
2660
2661 static void
2662 mark_reg_gr_used_mask (rtx reg, void *data ATTRIBUTE_UNUSED)
2663 {
2664   unsigned int regno = REGNO (reg);
2665   if (regno < 32)
2666     {
2667       unsigned int i, n = REG_NREGS (reg);
2668       for (i = 0; i < n; ++i)
2669         current_frame_info.gr_used_mask |= 1 << (regno + i);
2670     }
2671 }
2672
2673
2674 /* Returns the number of bytes offset between the frame pointer and the stack
2675    pointer for the current function.  SIZE is the number of bytes of space
2676    needed for local variables.  */
2677
2678 static void
2679 ia64_compute_frame_size (HOST_WIDE_INT size)
2680 {
2681   HOST_WIDE_INT total_size;
2682   HOST_WIDE_INT spill_size = 0;
2683   HOST_WIDE_INT extra_spill_size = 0;
2684   HOST_WIDE_INT pretend_args_size;
2685   HARD_REG_SET mask;
2686   int n_spilled = 0;
2687   int spilled_gr_p = 0;
2688   int spilled_fr_p = 0;
2689   unsigned int regno;
2690   int min_regno;
2691   int max_regno;
2692   int i;
2693
2694   if (current_frame_info.initialized)
2695     return;
2696
2697   memset (&current_frame_info, 0, sizeof current_frame_info);
2698   CLEAR_HARD_REG_SET (mask);
2699
2700   /* Don't allocate scratches to the return register.  */
2701   diddle_return_value (mark_reg_gr_used_mask, NULL);
2702
2703   /* Don't allocate scratches to the EH scratch registers.  */
2704   if (cfun->machine->ia64_eh_epilogue_sp)
2705     mark_reg_gr_used_mask (cfun->machine->ia64_eh_epilogue_sp, NULL);
2706   if (cfun->machine->ia64_eh_epilogue_bsp)
2707     mark_reg_gr_used_mask (cfun->machine->ia64_eh_epilogue_bsp, NULL);
2708
2709   /* Static stack checking uses r2 and r3.  */
2710   if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK
2711       || flag_stack_clash_protection)
2712     current_frame_info.gr_used_mask |= 0xc;
2713
2714   /* Find the size of the register stack frame.  We have only 80 local
2715      registers, because we reserve 8 for the inputs and 8 for the
2716      outputs.  */
2717
2718   /* Skip HARD_FRAME_POINTER_REGNUM (loc79) when frame_pointer_needed,
2719      since we'll be adjusting that down later.  */
2720   regno = LOC_REG (78) + ! frame_pointer_needed;
2721   for (; regno >= LOC_REG (0); regno--)
2722     if (df_regs_ever_live_p (regno) && !is_emitted (regno))
2723       break;
2724   current_frame_info.n_local_regs = regno - LOC_REG (0) + 1;
2725
2726   /* For functions marked with the syscall_linkage attribute, we must mark
2727      all eight input registers as in use, so that locals aren't visible to
2728      the caller.  */
2729
2730   if (cfun->machine->n_varargs > 0
2731       || lookup_attribute ("syscall_linkage",
2732                            TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
2733     current_frame_info.n_input_regs = 8;
2734   else
2735     {
2736       for (regno = IN_REG (7); regno >= IN_REG (0); regno--)
2737         if (df_regs_ever_live_p (regno))
2738           break;
2739       current_frame_info.n_input_regs = regno - IN_REG (0) + 1;
2740     }
2741
2742   for (regno = OUT_REG (7); regno >= OUT_REG (0); regno--)
2743     if (df_regs_ever_live_p (regno))
2744       break;
2745   i = regno - OUT_REG (0) + 1;
2746
2747 #ifndef PROFILE_HOOK
2748   /* When -p profiling, we need one output register for the mcount argument.
2749      Likewise for -a profiling for the bb_init_func argument.  For -ax
2750      profiling, we need two output registers for the two bb_init_trace_func
2751      arguments.  */
2752   if (crtl->profile)
2753     i = MAX (i, 1);
2754 #endif
2755   current_frame_info.n_output_regs = i;
2756
2757   /* ??? No rotating register support yet.  */
2758   current_frame_info.n_rotate_regs = 0;
2759
2760   /* Discover which registers need spilling, and how much room that
2761      will take.  Begin with floating point and general registers,
2762      which will always wind up on the stack.  */
2763
2764   for (regno = FR_REG (2); regno <= FR_REG (127); regno++)
2765     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2766       {
2767         SET_HARD_REG_BIT (mask, regno);
2768         spill_size += 16;
2769         n_spilled += 1;
2770         spilled_fr_p = 1;
2771       }
2772
2773   for (regno = GR_REG (1); regno <= GR_REG (31); regno++)
2774     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2775       {
2776         SET_HARD_REG_BIT (mask, regno);
2777         spill_size += 8;
2778         n_spilled += 1;
2779         spilled_gr_p = 1;
2780       }
2781
2782   for (regno = BR_REG (1); regno <= BR_REG (7); regno++)
2783     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2784       {
2785         SET_HARD_REG_BIT (mask, regno);
2786         spill_size += 8;
2787         n_spilled += 1;
2788       }
2789
2790   /* Now come all special registers that might get saved in other
2791      general registers.  */
2792
2793   if (frame_pointer_needed)
2794     {
2795       current_frame_info.r[reg_fp] = find_gr_spill (reg_fp, 1);
2796       /* If we did not get a register, then we take LOC79.  This is guaranteed
2797          to be free, even if regs_ever_live is already set, because this is
2798          HARD_FRAME_POINTER_REGNUM.  This requires incrementing n_local_regs,
2799          as we don't count loc79 above.  */
2800       if (current_frame_info.r[reg_fp] == 0)
2801         {
2802           current_frame_info.r[reg_fp] = LOC_REG (79);
2803           current_frame_info.n_local_regs = LOC_REG (79) - LOC_REG (0) + 1;
2804         }
2805     }
2806
2807   if (! crtl->is_leaf)
2808     {
2809       /* Emit a save of BR0 if we call other functions.  Do this even
2810          if this function doesn't return, as EH depends on this to be
2811          able to unwind the stack.  */
2812       SET_HARD_REG_BIT (mask, BR_REG (0));
2813
2814       current_frame_info.r[reg_save_b0] = find_gr_spill (reg_save_b0, 1);
2815       if (current_frame_info.r[reg_save_b0] == 0)
2816         {
2817           extra_spill_size += 8;
2818           n_spilled += 1;
2819         }
2820
2821       /* Similarly for ar.pfs.  */
2822       SET_HARD_REG_BIT (mask, AR_PFS_REGNUM);
2823       current_frame_info.r[reg_save_ar_pfs] = find_gr_spill (reg_save_ar_pfs, 1);
2824       if (current_frame_info.r[reg_save_ar_pfs] == 0)
2825         {
2826           extra_spill_size += 8;
2827           n_spilled += 1;
2828         }
2829
2830       /* Similarly for gp.  Note that if we're calling setjmp, the stacked
2831          registers are clobbered, so we fall back to the stack.  */
2832       current_frame_info.r[reg_save_gp]
2833         = (cfun->calls_setjmp ? 0 : find_gr_spill (reg_save_gp, 1));
2834       if (current_frame_info.r[reg_save_gp] == 0)
2835         {
2836           SET_HARD_REG_BIT (mask, GR_REG (1));
2837           spill_size += 8;
2838           n_spilled += 1;
2839         }
2840     }
2841   else
2842     {
2843       if (df_regs_ever_live_p (BR_REG (0)) && ! call_used_regs[BR_REG (0)])
2844         {
2845           SET_HARD_REG_BIT (mask, BR_REG (0));
2846           extra_spill_size += 8;
2847           n_spilled += 1;
2848         }
2849
2850       if (df_regs_ever_live_p (AR_PFS_REGNUM))
2851         {
2852           SET_HARD_REG_BIT (mask, AR_PFS_REGNUM);
2853           current_frame_info.r[reg_save_ar_pfs]
2854             = find_gr_spill (reg_save_ar_pfs, 1);
2855           if (current_frame_info.r[reg_save_ar_pfs] == 0)
2856             {
2857               extra_spill_size += 8;
2858               n_spilled += 1;
2859             }
2860         }
2861     }
2862
2863   /* Unwind descriptor hackery: things are most efficient if we allocate
2864      consecutive GR save registers for RP, PFS, FP in that order. However,
2865      it is absolutely critical that FP get the only hard register that's
2866      guaranteed to be free, so we allocated it first.  If all three did
2867      happen to be allocated hard regs, and are consecutive, rearrange them
2868      into the preferred order now.
2869
2870      If we have already emitted code for any of those registers,
2871      then it's already too late to change.  */
2872   min_regno = MIN (current_frame_info.r[reg_fp],
2873                    MIN (current_frame_info.r[reg_save_b0],
2874                         current_frame_info.r[reg_save_ar_pfs]));
2875   max_regno = MAX (current_frame_info.r[reg_fp],
2876                    MAX (current_frame_info.r[reg_save_b0],
2877                         current_frame_info.r[reg_save_ar_pfs]));
2878   if (min_regno > 0
2879       && min_regno + 2 == max_regno
2880       && (current_frame_info.r[reg_fp] == min_regno + 1
2881           || current_frame_info.r[reg_save_b0] == min_regno + 1
2882           || current_frame_info.r[reg_save_ar_pfs] == min_regno + 1)
2883       && (emitted_frame_related_regs[reg_save_b0] == 0
2884           || emitted_frame_related_regs[reg_save_b0] == min_regno)
2885       && (emitted_frame_related_regs[reg_save_ar_pfs] == 0
2886           || emitted_frame_related_regs[reg_save_ar_pfs] == min_regno + 1)
2887       && (emitted_frame_related_regs[reg_fp] == 0
2888           || emitted_frame_related_regs[reg_fp] == min_regno + 2))
2889     {
2890       current_frame_info.r[reg_save_b0] = min_regno;
2891       current_frame_info.r[reg_save_ar_pfs] = min_regno + 1;
2892       current_frame_info.r[reg_fp] = min_regno + 2;
2893     }
2894
2895   /* See if we need to store the predicate register block.  */
2896   for (regno = PR_REG (0); regno <= PR_REG (63); regno++)
2897     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2898       break;
2899   if (regno <= PR_REG (63))
2900     {
2901       SET_HARD_REG_BIT (mask, PR_REG (0));
2902       current_frame_info.r[reg_save_pr] = find_gr_spill (reg_save_pr, 1);
2903       if (current_frame_info.r[reg_save_pr] == 0)
2904         {
2905           extra_spill_size += 8;
2906           n_spilled += 1;
2907         }
2908
2909       /* ??? Mark them all as used so that register renaming and such
2910          are free to use them.  */
2911       for (regno = PR_REG (0); regno <= PR_REG (63); regno++)
2912         df_set_regs_ever_live (regno, true);
2913     }
2914
2915   /* If we're forced to use st8.spill, we're forced to save and restore
2916      ar.unat as well.  The check for existing liveness allows inline asm
2917      to touch ar.unat.  */
2918   if (spilled_gr_p || cfun->machine->n_varargs
2919       || df_regs_ever_live_p (AR_UNAT_REGNUM))
2920     {
2921       df_set_regs_ever_live (AR_UNAT_REGNUM, true);
2922       SET_HARD_REG_BIT (mask, AR_UNAT_REGNUM);
2923       current_frame_info.r[reg_save_ar_unat]
2924         = find_gr_spill (reg_save_ar_unat, spill_size == 0);
2925       if (current_frame_info.r[reg_save_ar_unat] == 0)
2926         {
2927           extra_spill_size += 8;
2928           n_spilled += 1;
2929         }
2930     }
2931
2932   if (df_regs_ever_live_p (AR_LC_REGNUM))
2933     {
2934       SET_HARD_REG_BIT (mask, AR_LC_REGNUM);
2935       current_frame_info.r[reg_save_ar_lc]
2936         = find_gr_spill (reg_save_ar_lc, spill_size == 0);
2937       if (current_frame_info.r[reg_save_ar_lc] == 0)
2938         {
2939           extra_spill_size += 8;
2940           n_spilled += 1;
2941         }
2942     }
2943
2944   /* If we have an odd number of words of pretend arguments written to
2945      the stack, then the FR save area will be unaligned.  We round the
2946      size of this area up to keep things 16 byte aligned.  */
2947   if (spilled_fr_p)
2948     pretend_args_size = IA64_STACK_ALIGN (crtl->args.pretend_args_size);
2949   else
2950     pretend_args_size = crtl->args.pretend_args_size;
2951
2952   total_size = (spill_size + extra_spill_size + size + pretend_args_size
2953                 + crtl->outgoing_args_size);
2954   total_size = IA64_STACK_ALIGN (total_size);
2955
2956   /* We always use the 16-byte scratch area provided by the caller, but
2957      if we are a leaf function, there's no one to which we need to provide
2958      a scratch area.  However, if the function allocates dynamic stack space,
2959      the dynamic offset is computed early and contains STACK_POINTER_OFFSET,
2960      so we need to cope.  */
2961   if (crtl->is_leaf && !cfun->calls_alloca)
2962     total_size = MAX (0, total_size - 16);
2963
2964   current_frame_info.total_size = total_size;
2965   current_frame_info.spill_cfa_off = pretend_args_size - 16;
2966   current_frame_info.spill_size = spill_size;
2967   current_frame_info.extra_spill_size = extra_spill_size;
2968   COPY_HARD_REG_SET (current_frame_info.mask, mask);
2969   current_frame_info.n_spilled = n_spilled;
2970   current_frame_info.initialized = reload_completed;
2971 }
2972
2973 /* Worker function for TARGET_CAN_ELIMINATE.  */
2974
2975 bool
2976 ia64_can_eliminate (const int from ATTRIBUTE_UNUSED, const int to)
2977 {
2978   return (to == BR_REG (0) ? crtl->is_leaf : true);
2979 }
2980
2981 /* Compute the initial difference between the specified pair of registers.  */
2982
2983 HOST_WIDE_INT
2984 ia64_initial_elimination_offset (int from, int to)
2985 {
2986   HOST_WIDE_INT offset;
2987
2988   ia64_compute_frame_size (get_frame_size ());
2989   switch (from)
2990     {
2991     case FRAME_POINTER_REGNUM:
2992       switch (to)
2993         {
2994         case HARD_FRAME_POINTER_REGNUM:
2995           offset = -current_frame_info.total_size;
2996           if (!crtl->is_leaf || cfun->calls_alloca)
2997             offset += 16 + crtl->outgoing_args_size;
2998           break;
2999
3000         case STACK_POINTER_REGNUM:
3001           offset = 0;
3002           if (!crtl->is_leaf || cfun->calls_alloca)
3003             offset += 16 + crtl->outgoing_args_size;
3004           break;
3005
3006         default:
3007           gcc_unreachable ();
3008         }
3009       break;
3010
3011     case ARG_POINTER_REGNUM:
3012       /* Arguments start above the 16 byte save area, unless stdarg
3013          in which case we store through the 16 byte save area.  */
3014       switch (to)
3015         {
3016         case HARD_FRAME_POINTER_REGNUM:
3017           offset = 16 - crtl->args.pretend_args_size;
3018           break;
3019
3020         case STACK_POINTER_REGNUM:
3021           offset = (current_frame_info.total_size
3022                     + 16 - crtl->args.pretend_args_size);
3023           break;
3024
3025         default:
3026           gcc_unreachable ();
3027         }
3028       break;
3029
3030     default:
3031       gcc_unreachable ();
3032     }
3033
3034   return offset;
3035 }
3036
3037 /* If there are more than a trivial number of register spills, we use
3038    two interleaved iterators so that we can get two memory references
3039    per insn group.
3040
3041    In order to simplify things in the prologue and epilogue expanders,
3042    we use helper functions to fix up the memory references after the
3043    fact with the appropriate offsets to a POST_MODIFY memory mode.
3044    The following data structure tracks the state of the two iterators
3045    while insns are being emitted.  */
3046
3047 struct spill_fill_data
3048 {
3049   rtx_insn *init_after;         /* point at which to emit initializations */
3050   rtx init_reg[2];              /* initial base register */
3051   rtx iter_reg[2];              /* the iterator registers */
3052   rtx *prev_addr[2];            /* address of last memory use */
3053   rtx_insn *prev_insn[2];       /* the insn corresponding to prev_addr */
3054   HOST_WIDE_INT prev_off[2];    /* last offset */
3055   int n_iter;                   /* number of iterators in use */
3056   int next_iter;                /* next iterator to use */
3057   unsigned int save_gr_used_mask;
3058 };
3059
3060 static struct spill_fill_data spill_fill_data;
3061
3062 static void
3063 setup_spill_pointers (int n_spills, rtx init_reg, HOST_WIDE_INT cfa_off)
3064 {
3065   int i;
3066
3067   spill_fill_data.init_after = get_last_insn ();
3068   spill_fill_data.init_reg[0] = init_reg;
3069   spill_fill_data.init_reg[1] = init_reg;
3070   spill_fill_data.prev_addr[0] = NULL;
3071   spill_fill_data.prev_addr[1] = NULL;
3072   spill_fill_data.prev_insn[0] = NULL;
3073   spill_fill_data.prev_insn[1] = NULL;
3074   spill_fill_data.prev_off[0] = cfa_off;
3075   spill_fill_data.prev_off[1] = cfa_off;
3076   spill_fill_data.next_iter = 0;
3077   spill_fill_data.save_gr_used_mask = current_frame_info.gr_used_mask;
3078
3079   spill_fill_data.n_iter = 1 + (n_spills > 2);
3080   for (i = 0; i < spill_fill_data.n_iter; ++i)
3081     {
3082       int regno = next_scratch_gr_reg ();
3083       spill_fill_data.iter_reg[i] = gen_rtx_REG (DImode, regno);
3084       current_frame_info.gr_used_mask |= 1 << regno;
3085     }
3086 }
3087
3088 static void
3089 finish_spill_pointers (void)
3090 {
3091   current_frame_info.gr_used_mask = spill_fill_data.save_gr_used_mask;
3092 }
3093
3094 static rtx
3095 spill_restore_mem (rtx reg, HOST_WIDE_INT cfa_off)
3096 {
3097   int iter = spill_fill_data.next_iter;
3098   HOST_WIDE_INT disp = spill_fill_data.prev_off[iter] - cfa_off;
3099   rtx disp_rtx = GEN_INT (disp);
3100   rtx mem;
3101
3102   if (spill_fill_data.prev_addr[iter])
3103     {
3104       if (satisfies_constraint_N (disp_rtx))
3105         {
3106           *spill_fill_data.prev_addr[iter]
3107             = gen_rtx_POST_MODIFY (DImode, spill_fill_data.iter_reg[iter],
3108                                    gen_rtx_PLUS (DImode,
3109                                                  spill_fill_data.iter_reg[iter],
3110                                                  disp_rtx));
3111           add_reg_note (spill_fill_data.prev_insn[iter],
3112                         REG_INC, spill_fill_data.iter_reg[iter]);
3113         }
3114       else
3115         {
3116           /* ??? Could use register post_modify for loads.  */
3117           if (!satisfies_constraint_I (disp_rtx))
3118             {
3119               rtx tmp = gen_rtx_REG (DImode, next_scratch_gr_reg ());
3120               emit_move_insn (tmp, disp_rtx);
3121               disp_rtx = tmp;
3122             }
3123           emit_insn (gen_adddi3 (spill_fill_data.iter_reg[iter],
3124                                  spill_fill_data.iter_reg[iter], disp_rtx));
3125         }
3126     }
3127   /* Micro-optimization: if we've created a frame pointer, it's at
3128      CFA 0, which may allow the real iterator to be initialized lower,
3129      slightly increasing parallelism.  Also, if there are few saves
3130      it may eliminate the iterator entirely.  */
3131   else if (disp == 0
3132            && spill_fill_data.init_reg[iter] == stack_pointer_rtx
3133            && frame_pointer_needed)
3134     {
3135       mem = gen_rtx_MEM (GET_MODE (reg), hard_frame_pointer_rtx);
3136       set_mem_alias_set (mem, get_varargs_alias_set ());
3137       return mem;
3138     }
3139   else
3140     {
3141       rtx seq;
3142       rtx_insn *insn;
3143
3144       if (disp == 0)
3145         seq = gen_movdi (spill_fill_data.iter_reg[iter],
3146                          spill_fill_data.init_reg[iter]);
3147       else
3148         {
3149           start_sequence ();
3150
3151           if (!satisfies_constraint_I (disp_rtx))
3152             {
3153               rtx tmp = gen_rtx_REG (DImode, next_scratch_gr_reg ());
3154               emit_move_insn (tmp, disp_rtx);
3155               disp_rtx = tmp;
3156             }
3157
3158           emit_insn (gen_adddi3 (spill_fill_data.iter_reg[iter],
3159                                  spill_fill_data.init_reg[iter],
3160                                  disp_rtx));
3161
3162           seq = get_insns ();
3163           end_sequence ();
3164         }
3165
3166       /* Careful for being the first insn in a sequence.  */
3167       if (spill_fill_data.init_after)
3168         insn = emit_insn_after (seq, spill_fill_data.init_after);
3169       else
3170         {
3171           rtx_insn *first = get_insns ();
3172           if (first)
3173             insn = emit_insn_before (seq, first);
3174           else
3175             insn = emit_insn (seq);
3176         }
3177       spill_fill_data.init_after = insn;
3178     }
3179
3180   mem = gen_rtx_MEM (GET_MODE (reg), spill_fill_data.iter_reg[iter]);
3181
3182   /* ??? Not all of the spills are for varargs, but some of them are.
3183      The rest of the spills belong in an alias set of their own.  But
3184      it doesn't actually hurt to include them here.  */
3185   set_mem_alias_set (mem, get_varargs_alias_set ());
3186
3187   spill_fill_data.prev_addr[iter] = &XEXP (mem, 0);
3188   spill_fill_data.prev_off[iter] = cfa_off;
3189
3190   if (++iter >= spill_fill_data.n_iter)
3191     iter = 0;
3192   spill_fill_data.next_iter = iter;
3193
3194   return mem;
3195 }
3196
3197 static void
3198 do_spill (rtx (*move_fn) (rtx, rtx, rtx), rtx reg, HOST_WIDE_INT cfa_off,
3199           rtx frame_reg)
3200 {
3201   int iter = spill_fill_data.next_iter;
3202   rtx mem;
3203   rtx_insn *insn;
3204
3205   mem = spill_restore_mem (reg, cfa_off);
3206   insn = emit_insn ((*move_fn) (mem, reg, GEN_INT (cfa_off)));
3207   spill_fill_data.prev_insn[iter] = insn;
3208
3209   if (frame_reg)
3210     {
3211       rtx base;
3212       HOST_WIDE_INT off;
3213
3214       RTX_FRAME_RELATED_P (insn) = 1;
3215
3216       /* Don't even pretend that the unwind code can intuit its way
3217          through a pair of interleaved post_modify iterators.  Just
3218          provide the correct answer.  */
3219
3220       if (frame_pointer_needed)
3221         {
3222           base = hard_frame_pointer_rtx;
3223           off = - cfa_off;
3224         }
3225       else
3226         {
3227           base = stack_pointer_rtx;
3228           off = current_frame_info.total_size - cfa_off;
3229         }
3230
3231       add_reg_note (insn, REG_CFA_OFFSET,
3232                     gen_rtx_SET (gen_rtx_MEM (GET_MODE (reg),
3233                                               plus_constant (Pmode,
3234                                                              base, off)),
3235                                  frame_reg));
3236     }
3237 }
3238
3239 static void
3240 do_restore (rtx (*move_fn) (rtx, rtx, rtx), rtx reg, HOST_WIDE_INT cfa_off)
3241 {
3242   int iter = spill_fill_data.next_iter;
3243   rtx_insn *insn;
3244
3245   insn = emit_insn ((*move_fn) (reg, spill_restore_mem (reg, cfa_off),
3246                                 GEN_INT (cfa_off)));
3247   spill_fill_data.prev_insn[iter] = insn;
3248 }
3249
3250 /* Wrapper functions that discards the CONST_INT spill offset.  These
3251    exist so that we can give gr_spill/gr_fill the offset they need and
3252    use a consistent function interface.  */
3253
3254 static rtx
3255 gen_movdi_x (rtx dest, rtx src, rtx offset ATTRIBUTE_UNUSED)
3256 {
3257   return gen_movdi (dest, src);
3258 }
3259
3260 static rtx
3261 gen_fr_spill_x (rtx dest, rtx src, rtx offset ATTRIBUTE_UNUSED)
3262 {
3263   return gen_fr_spill (dest, src);
3264 }
3265
3266 static rtx
3267 gen_fr_restore_x (rtx dest, rtx src, rtx offset ATTRIBUTE_UNUSED)
3268 {
3269   return gen_fr_restore (dest, src);
3270 }
3271
3272 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
3273
3274 /* See Table 6.2 of the IA-64 Software Developer Manual, Volume 2.  */
3275 #define BACKING_STORE_SIZE(N) ((N) > 0 ? ((N) + (N)/63 + 1) * 8 : 0)
3276
3277 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
3278    inclusive.  These are offsets from the current stack pointer.  BS_SIZE
3279    is the size of the backing store.  ??? This clobbers r2 and r3.  */
3280
3281 static void
3282 ia64_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size,
3283                              int bs_size)
3284 {
3285   rtx r2 = gen_rtx_REG (Pmode, GR_REG (2));
3286   rtx r3 = gen_rtx_REG (Pmode, GR_REG (3));
3287   rtx p6 = gen_rtx_REG (BImode, PR_REG (6));
3288
3289   /* On the IA-64 there is a second stack in memory, namely the Backing Store
3290      of the Register Stack Engine.  We also need to probe it after checking
3291      that the 2 stacks don't overlap.  */
3292   emit_insn (gen_bsp_value (r3));
3293   emit_move_insn (r2, GEN_INT (-(first + size)));
3294
3295   /* Compare current value of BSP and SP registers.  */
3296   emit_insn (gen_rtx_SET (p6, gen_rtx_fmt_ee (LTU, BImode,
3297                                               r3, stack_pointer_rtx)));
3298
3299   /* Compute the address of the probe for the Backing Store (which grows
3300      towards higher addresses).  We probe only at the first offset of
3301      the next page because some OS (eg Linux/ia64) only extend the
3302      backing store when this specific address is hit (but generate a SEGV
3303      on other address).  Page size is the worst case (4KB).  The reserve
3304      size is at least 4096 - (96 + 2) * 8 = 3312 bytes, which is enough.
3305      Also compute the address of the last probe for the memory stack
3306      (which grows towards lower addresses).  */
3307   emit_insn (gen_rtx_SET (r3, plus_constant (Pmode, r3, 4095)));
3308   emit_insn (gen_rtx_SET (r2, gen_rtx_PLUS (Pmode, stack_pointer_rtx, r2)));
3309
3310   /* Compare them and raise SEGV if the former has topped the latter.  */
3311   emit_insn (gen_rtx_COND_EXEC (VOIDmode,
3312                                 gen_rtx_fmt_ee (NE, VOIDmode, p6, const0_rtx),
3313                                 gen_rtx_SET (p6, gen_rtx_fmt_ee (GEU, BImode,
3314                                                                  r3, r2))));
3315   emit_insn (gen_rtx_SET (gen_rtx_ZERO_EXTRACT (DImode, r3, GEN_INT (12),
3316                                                 const0_rtx),
3317                           const0_rtx));
3318   emit_insn (gen_rtx_COND_EXEC (VOIDmode,
3319                                 gen_rtx_fmt_ee (NE, VOIDmode, p6, const0_rtx),
3320                                 gen_rtx_TRAP_IF (VOIDmode, const1_rtx,
3321                                                  GEN_INT (11))));
3322
3323   /* Probe the Backing Store if necessary.  */
3324   if (bs_size > 0)
3325     emit_stack_probe (r3);
3326
3327   /* Probe the memory stack if necessary.  */
3328   if (size == 0)
3329     ;
3330
3331   /* See if we have a constant small number of probes to generate.  If so,
3332      that's the easy case.  */
3333   else if (size <= PROBE_INTERVAL)
3334     emit_stack_probe (r2);
3335
3336   /* The run-time loop is made up of 9 insns in the generic case while this
3337      compile-time loop is made up of 5+2*(n-2) insns for n # of intervals.  */
3338   else if (size <= 4 * PROBE_INTERVAL)
3339     {
3340       HOST_WIDE_INT i;
3341
3342       emit_move_insn (r2, GEN_INT (-(first + PROBE_INTERVAL)));
3343       emit_insn (gen_rtx_SET (r2,
3344                               gen_rtx_PLUS (Pmode, stack_pointer_rtx, r2)));
3345       emit_stack_probe (r2);
3346
3347       /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
3348          it exceeds SIZE.  If only two probes are needed, this will not
3349          generate any code.  Then probe at FIRST + SIZE.  */
3350       for (i = 2 * PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
3351         {
3352           emit_insn (gen_rtx_SET (r2,
3353                                   plus_constant (Pmode, r2, -PROBE_INTERVAL)));
3354           emit_stack_probe (r2);
3355         }
3356
3357       emit_insn (gen_rtx_SET (r2,
3358                               plus_constant (Pmode, r2,
3359                                              (i - PROBE_INTERVAL) - size)));
3360       emit_stack_probe (r2);
3361     }
3362
3363   /* Otherwise, do the same as above, but in a loop.  Note that we must be
3364      extra careful with variables wrapping around because we might be at
3365      the very top (or the very bottom) of the address space and we have
3366      to be able to handle this case properly; in particular, we use an
3367      equality test for the loop condition.  */
3368   else
3369     {
3370       HOST_WIDE_INT rounded_size;
3371
3372       emit_move_insn (r2, GEN_INT (-first));
3373
3374
3375       /* Step 1: round SIZE to the previous multiple of the interval.  */
3376
3377       rounded_size = size & -PROBE_INTERVAL;
3378
3379
3380       /* Step 2: compute initial and final value of the loop counter.  */
3381
3382       /* TEST_ADDR = SP + FIRST.  */
3383       emit_insn (gen_rtx_SET (r2,
3384                               gen_rtx_PLUS (Pmode, stack_pointer_rtx, r2)));
3385
3386       /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE.  */
3387       if (rounded_size > (1 << 21))
3388         {
3389           emit_move_insn (r3, GEN_INT (-rounded_size));
3390           emit_insn (gen_rtx_SET (r3, gen_rtx_PLUS (Pmode, r2, r3)));
3391         }
3392       else
3393         emit_insn (gen_rtx_SET (r3, gen_rtx_PLUS (Pmode, r2,
3394                                                   GEN_INT (-rounded_size))));
3395
3396
3397       /* Step 3: the loop
3398
3399          do
3400            {
3401              TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
3402              probe at TEST_ADDR
3403            }
3404          while (TEST_ADDR != LAST_ADDR)
3405
3406          probes at FIRST + N * PROBE_INTERVAL for values of N from 1
3407          until it is equal to ROUNDED_SIZE.  */
3408
3409       emit_insn (gen_probe_stack_range (r2, r2, r3));
3410
3411
3412       /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
3413          that SIZE is equal to ROUNDED_SIZE.  */
3414
3415       /* TEMP = SIZE - ROUNDED_SIZE.  */
3416       if (size != rounded_size)
3417         {
3418           emit_insn (gen_rtx_SET (r2, plus_constant (Pmode, r2,
3419                                                      rounded_size - size)));
3420           emit_stack_probe (r2);
3421         }
3422     }
3423
3424   /* Make sure nothing is scheduled before we are done.  */
3425   emit_insn (gen_blockage ());
3426 }
3427
3428 /* Probe a range of stack addresses from REG1 to REG2 inclusive.  These are
3429    absolute addresses.  */
3430
3431 const char *
3432 output_probe_stack_range (rtx reg1, rtx reg2)
3433 {
3434   static int labelno = 0;
3435   char loop_lab[32];
3436   rtx xops[3];
3437
3438   ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno++);
3439
3440   /* Loop.  */
3441   ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
3442
3443   /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL.  */
3444   xops[0] = reg1;
3445   xops[1] = GEN_INT (-PROBE_INTERVAL);
3446   output_asm_insn ("addl %0 = %1, %0", xops);
3447   fputs ("\t;;\n", asm_out_file);
3448
3449   /* Probe at TEST_ADDR.  */
3450   output_asm_insn ("probe.w.fault %0, 0", xops);
3451
3452   /* Test if TEST_ADDR == LAST_ADDR.  */
3453   xops[1] = reg2;
3454   xops[2] = gen_rtx_REG (BImode, PR_REG (6));
3455   output_asm_insn ("cmp.eq %2, %I2 = %0, %1", xops);
3456
3457   /* Branch.  */
3458   fprintf (asm_out_file, "\t(%s) br.cond.dpnt ", reg_names [PR_REG (7)]);
3459   assemble_name_raw (asm_out_file, loop_lab);
3460   fputc ('\n', asm_out_file);
3461
3462   return "";
3463 }
3464
3465 /* Called after register allocation to add any instructions needed for the
3466    prologue.  Using a prologue insn is favored compared to putting all of the
3467    instructions in output_function_prologue(), since it allows the scheduler
3468    to intermix instructions with the saves of the caller saved registers.  In
3469    some cases, it might be necessary to emit a barrier instruction as the last
3470    insn to prevent such scheduling.
3471
3472    Also any insns generated here should have RTX_FRAME_RELATED_P(insn) = 1
3473    so that the debug info generation code can handle them properly.
3474
3475    The register save area is laid out like so:
3476    cfa+16
3477         [ varargs spill area ]
3478         [ fr register spill area ]
3479         [ br register spill area ]
3480         [ ar register spill area ]
3481         [ pr register spill area ]
3482         [ gr register spill area ] */
3483
3484 /* ??? Get inefficient code when the frame size is larger than can fit in an
3485    adds instruction.  */
3486
3487 void
3488 ia64_expand_prologue (void)
3489 {
3490   rtx_insn *insn;
3491   rtx ar_pfs_save_reg, ar_unat_save_reg;
3492   int i, epilogue_p, regno, alt_regno, cfa_off, n_varargs;
3493   rtx reg, alt_reg;
3494
3495   ia64_compute_frame_size (get_frame_size ());
3496   last_scratch_gr_reg = 15;
3497
3498   if (flag_stack_usage_info)
3499     current_function_static_stack_size = current_frame_info.total_size;
3500
3501   if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK
3502       || flag_stack_clash_protection)
3503     {
3504       HOST_WIDE_INT size = current_frame_info.total_size;
3505       int bs_size = BACKING_STORE_SIZE (current_frame_info.n_input_regs
3506                                           + current_frame_info.n_local_regs);
3507
3508       if (crtl->is_leaf && !cfun->calls_alloca)
3509         {
3510           if (size > PROBE_INTERVAL && size > get_stack_check_protect ())
3511             ia64_emit_probe_stack_range (get_stack_check_protect (),
3512                                          size - get_stack_check_protect (),
3513                                          bs_size);
3514           else if (size + bs_size > get_stack_check_protect ())
3515             ia64_emit_probe_stack_range (get_stack_check_protect (),
3516                                          0, bs_size);
3517         }
3518       else if (size + bs_size > 0)
3519         ia64_emit_probe_stack_range (get_stack_check_protect (), size, bs_size);
3520     }
3521
3522   if (dump_file)
3523     {
3524       fprintf (dump_file, "ia64 frame related registers "
3525                "recorded in current_frame_info.r[]:\n");
3526 #define PRINTREG(a) if (current_frame_info.r[a]) \
3527         fprintf(dump_file, "%s = %d\n", #a, current_frame_info.r[a])
3528       PRINTREG(reg_fp);
3529       PRINTREG(reg_save_b0);
3530       PRINTREG(reg_save_pr);
3531       PRINTREG(reg_save_ar_pfs);
3532       PRINTREG(reg_save_ar_unat);
3533       PRINTREG(reg_save_ar_lc);
3534       PRINTREG(reg_save_gp);
3535 #undef PRINTREG
3536     }
3537
3538   /* If there is no epilogue, then we don't need some prologue insns.
3539      We need to avoid emitting the dead prologue insns, because flow
3540      will complain about them.  */
3541   if (optimize)
3542     {
3543       edge e;
3544       edge_iterator ei;
3545
3546       FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
3547         if ((e->flags & EDGE_FAKE) == 0
3548             && (e->flags & EDGE_FALLTHRU) != 0)
3549           break;
3550       epilogue_p = (e != NULL);
3551     }
3552   else
3553     epilogue_p = 1;
3554
3555   /* Set the local, input, and output register names.  We need to do this
3556      for GNU libc, which creates crti.S/crtn.S by splitting initfini.c in
3557      half.  If we use in/loc/out register names, then we get assembler errors
3558      in crtn.S because there is no alloc insn or regstk directive in there.  */
3559   if (! TARGET_REG_NAMES)
3560     {
3561       int inputs = current_frame_info.n_input_regs;
3562       int locals = current_frame_info.n_local_regs;
3563       int outputs = current_frame_info.n_output_regs;
3564
3565       for (i = 0; i < inputs; i++)
3566         reg_names[IN_REG (i)] = ia64_reg_numbers[i];
3567       for (i = 0; i < locals; i++)
3568         reg_names[LOC_REG (i)] = ia64_reg_numbers[inputs + i];
3569       for (i = 0; i < outputs; i++)
3570         reg_names[OUT_REG (i)] = ia64_reg_numbers[inputs + locals + i];
3571     }
3572
3573   /* Set the frame pointer register name.  The regnum is logically loc79,
3574      but of course we'll not have allocated that many locals.  Rather than
3575      worrying about renumbering the existing rtxs, we adjust the name.  */
3576   /* ??? This code means that we can never use one local register when
3577      there is a frame pointer.  loc79 gets wasted in this case, as it is
3578      renamed to a register that will never be used.  See also the try_locals
3579      code in find_gr_spill.  */
3580   if (current_frame_info.r[reg_fp])
3581     {
3582       const char *tmp = reg_names[HARD_FRAME_POINTER_REGNUM];
3583       reg_names[HARD_FRAME_POINTER_REGNUM]
3584         = reg_names[current_frame_info.r[reg_fp]];
3585       reg_names[current_frame_info.r[reg_fp]] = tmp;
3586     }
3587
3588   /* We don't need an alloc instruction if we've used no outputs or locals.  */
3589   if (current_frame_info.n_local_regs == 0
3590       && current_frame_info.n_output_regs == 0
3591       && current_frame_info.n_input_regs <= crtl->args.info.int_regs
3592       && !TEST_HARD_REG_BIT (current_frame_info.mask, AR_PFS_REGNUM))
3593     {
3594       /* If there is no alloc, but there are input registers used, then we
3595          need a .regstk directive.  */
3596       current_frame_info.need_regstk = (TARGET_REG_NAMES != 0);
3597       ar_pfs_save_reg = NULL_RTX;
3598     }
3599   else
3600     {
3601       current_frame_info.need_regstk = 0;
3602
3603       if (current_frame_info.r[reg_save_ar_pfs])
3604         {
3605           regno = current_frame_info.r[reg_save_ar_pfs];
3606           reg_emitted (reg_save_ar_pfs);
3607         }
3608       else
3609         regno = next_scratch_gr_reg ();
3610       ar_pfs_save_reg = gen_rtx_REG (DImode, regno);
3611
3612       insn = emit_insn (gen_alloc (ar_pfs_save_reg,
3613                                    GEN_INT (current_frame_info.n_input_regs),
3614                                    GEN_INT (current_frame_info.n_local_regs),
3615                                    GEN_INT (current_frame_info.n_output_regs),
3616                                    GEN_INT (current_frame_info.n_rotate_regs)));
3617       if (current_frame_info.r[reg_save_ar_pfs])
3618         {
3619           RTX_FRAME_RELATED_P (insn) = 1;
3620           add_reg_note (insn, REG_CFA_REGISTER,
3621                         gen_rtx_SET (ar_pfs_save_reg,
3622                                      gen_rtx_REG (DImode, AR_PFS_REGNUM)));
3623         }
3624     }
3625
3626   /* Set up frame pointer, stack pointer, and spill iterators.  */
3627
3628   n_varargs = cfun->machine->n_varargs;
3629   setup_spill_pointers (current_frame_info.n_spilled + n_varargs,
3630                         stack_pointer_rtx, 0);
3631
3632   if (frame_pointer_needed)
3633     {
3634       insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
3635       RTX_FRAME_RELATED_P (insn) = 1;
3636
3637       /* Force the unwind info to recognize this as defining a new CFA,
3638          rather than some temp register setup.  */
3639       add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL_RTX);
3640     }
3641
3642   if (current_frame_info.total_size != 0)
3643     {
3644       rtx frame_size_rtx = GEN_INT (- current_frame_info.total_size);
3645       rtx offset;
3646
3647       if (satisfies_constraint_I (frame_size_rtx))
3648         offset = frame_size_rtx;
3649       else
3650         {
3651           regno = next_scratch_gr_reg ();
3652           offset = gen_rtx_REG (DImode, regno);
3653           emit_move_insn (offset, frame_size_rtx);
3654         }
3655
3656       insn = emit_insn (gen_adddi3 (stack_pointer_rtx,
3657                                     stack_pointer_rtx, offset));
3658
3659       if (! frame_pointer_needed)
3660         {
3661           RTX_FRAME_RELATED_P (insn) = 1;
3662           add_reg_note (insn, REG_CFA_ADJUST_CFA,
3663                         gen_rtx_SET (stack_pointer_rtx,
3664                                      gen_rtx_PLUS (DImode,
3665                                                    stack_pointer_rtx,
3666                                                    frame_size_rtx)));
3667         }
3668
3669       /* ??? At this point we must generate a magic insn that appears to
3670          modify the stack pointer, the frame pointer, and all spill
3671          iterators.  This would allow the most scheduling freedom.  For
3672          now, just hard stop.  */
3673       emit_insn (gen_blockage ());
3674     }
3675
3676   /* Must copy out ar.unat before doing any integer spills.  */
3677   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM))
3678     {
3679       if (current_frame_info.r[reg_save_ar_unat])
3680         {
3681           ar_unat_save_reg
3682             = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_unat]);
3683           reg_emitted (reg_save_ar_unat);
3684         }
3685       else
3686         {
3687           alt_regno = next_scratch_gr_reg ();
3688           ar_unat_save_reg = gen_rtx_REG (DImode, alt_regno);
3689           current_frame_info.gr_used_mask |= 1 << alt_regno;
3690         }
3691
3692       reg = gen_rtx_REG (DImode, AR_UNAT_REGNUM);
3693       insn = emit_move_insn (ar_unat_save_reg, reg);
3694       if (current_frame_info.r[reg_save_ar_unat])
3695         {
3696           RTX_FRAME_RELATED_P (insn) = 1;
3697           add_reg_note (insn, REG_CFA_REGISTER, NULL_RTX);
3698         }
3699
3700       /* Even if we're not going to generate an epilogue, we still
3701          need to save the register so that EH works.  */
3702       if (! epilogue_p && current_frame_info.r[reg_save_ar_unat])
3703         emit_insn (gen_prologue_use (ar_unat_save_reg));
3704     }
3705   else
3706     ar_unat_save_reg = NULL_RTX;
3707
3708   /* Spill all varargs registers.  Do this before spilling any GR registers,
3709      since we want the UNAT bits for the GR registers to override the UNAT
3710      bits from varargs, which we don't care about.  */
3711
3712   cfa_off = -16;
3713   for (regno = GR_ARG_FIRST + 7; n_varargs > 0; --n_varargs, --regno)
3714     {
3715       reg = gen_rtx_REG (DImode, regno);
3716       do_spill (gen_gr_spill, reg, cfa_off += 8, NULL_RTX);
3717     }
3718
3719   /* Locate the bottom of the register save area.  */
3720   cfa_off = (current_frame_info.spill_cfa_off
3721              + current_frame_info.spill_size
3722              + current_frame_info.extra_spill_size);
3723
3724   /* Save the predicate register block either in a register or in memory.  */
3725   if (TEST_HARD_REG_BIT (current_frame_info.mask, PR_REG (0)))
3726     {
3727       reg = gen_rtx_REG (DImode, PR_REG (0));
3728       if (current_frame_info.r[reg_save_pr] != 0)
3729         {
3730           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_pr]);
3731           reg_emitted (reg_save_pr);
3732           insn = emit_move_insn (alt_reg, reg);
3733
3734           /* ??? Denote pr spill/fill by a DImode move that modifies all
3735              64 hard registers.  */
3736           RTX_FRAME_RELATED_P (insn) = 1;
3737           add_reg_note (insn, REG_CFA_REGISTER, NULL_RTX);
3738
3739           /* Even if we're not going to generate an epilogue, we still
3740              need to save the register so that EH works.  */
3741           if (! epilogue_p)
3742             emit_insn (gen_prologue_use (alt_reg));
3743         }
3744       else
3745         {
3746           alt_regno = next_scratch_gr_reg ();
3747           alt_reg = gen_rtx_REG (DImode, alt_regno);
3748           insn = emit_move_insn (alt_reg, reg);
3749           do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3750           cfa_off -= 8;
3751         }
3752     }
3753
3754   /* Handle AR regs in numerical order.  All of them get special handling.  */
3755   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM)
3756       && current_frame_info.r[reg_save_ar_unat] == 0)
3757     {
3758       reg = gen_rtx_REG (DImode, AR_UNAT_REGNUM);
3759       do_spill (gen_movdi_x, ar_unat_save_reg, cfa_off, reg);
3760       cfa_off -= 8;
3761     }
3762
3763   /* The alloc insn already copied ar.pfs into a general register.  The
3764      only thing we have to do now is copy that register to a stack slot
3765      if we'd not allocated a local register for the job.  */
3766   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_PFS_REGNUM)
3767       && current_frame_info.r[reg_save_ar_pfs] == 0)
3768     {
3769       reg = gen_rtx_REG (DImode, AR_PFS_REGNUM);
3770       do_spill (gen_movdi_x, ar_pfs_save_reg, cfa_off, reg);
3771       cfa_off -= 8;
3772     }
3773
3774   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_LC_REGNUM))
3775     {
3776       reg = gen_rtx_REG (DImode, AR_LC_REGNUM);
3777       if (current_frame_info.r[reg_save_ar_lc] != 0)
3778         {
3779           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_lc]);
3780           reg_emitted (reg_save_ar_lc);
3781           insn = emit_move_insn (alt_reg, reg);
3782           RTX_FRAME_RELATED_P (insn) = 1;
3783           add_reg_note (insn, REG_CFA_REGISTER, NULL_RTX);
3784
3785           /* Even if we're not going to generate an epilogue, we still
3786              need to save the register so that EH works.  */
3787           if (! epilogue_p)
3788             emit_insn (gen_prologue_use (alt_reg));
3789         }
3790       else
3791         {
3792           alt_regno = next_scratch_gr_reg ();
3793           alt_reg = gen_rtx_REG (DImode, alt_regno);
3794           emit_move_insn (alt_reg, reg);
3795           do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3796           cfa_off -= 8;
3797         }
3798     }
3799
3800   /* Save the return pointer.  */
3801   if (TEST_HARD_REG_BIT (current_frame_info.mask, BR_REG (0)))
3802     {
3803       reg = gen_rtx_REG (DImode, BR_REG (0));
3804       if (current_frame_info.r[reg_save_b0] != 0)
3805         {
3806           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_b0]);
3807           reg_emitted (reg_save_b0);
3808           insn = emit_move_insn (alt_reg, reg);
3809           RTX_FRAME_RELATED_P (insn) = 1;
3810           add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (alt_reg, pc_rtx));
3811
3812           /* Even if we're not going to generate an epilogue, we still
3813              need to save the register so that EH works.  */
3814           if (! epilogue_p)
3815             emit_insn (gen_prologue_use (alt_reg));
3816         }
3817       else
3818         {
3819           alt_regno = next_scratch_gr_reg ();
3820           alt_reg = gen_rtx_REG (DImode, alt_regno);
3821           emit_move_insn (alt_reg, reg);
3822           do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3823           cfa_off -= 8;
3824         }
3825     }
3826
3827   if (current_frame_info.r[reg_save_gp])
3828     {
3829       reg_emitted (reg_save_gp);
3830       insn = emit_move_insn (gen_rtx_REG (DImode,
3831                                           current_frame_info.r[reg_save_gp]),
3832                              pic_offset_table_rtx);
3833     }
3834
3835   /* We should now be at the base of the gr/br/fr spill area.  */
3836   gcc_assert (cfa_off == (current_frame_info.spill_cfa_off
3837                           + current_frame_info.spill_size));
3838
3839   /* Spill all general registers.  */
3840   for (regno = GR_REG (1); regno <= GR_REG (31); ++regno)
3841     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
3842       {
3843         reg = gen_rtx_REG (DImode, regno);
3844         do_spill (gen_gr_spill, reg, cfa_off, reg);
3845         cfa_off -= 8;
3846       }
3847
3848   /* Spill the rest of the BR registers.  */
3849   for (regno = BR_REG (1); regno <= BR_REG (7); ++regno)
3850     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
3851       {
3852         alt_regno = next_scratch_gr_reg ();
3853         alt_reg = gen_rtx_REG (DImode, alt_regno);
3854         reg = gen_rtx_REG (DImode, regno);
3855         emit_move_insn (alt_reg, reg);
3856         do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3857         cfa_off -= 8;
3858       }
3859
3860   /* Align the frame and spill all FR registers.  */
3861   for (regno = FR_REG (2); regno <= FR_REG (127); ++regno)
3862     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
3863       {
3864         gcc_assert (!(cfa_off & 15));
3865         reg = gen_rtx_REG (XFmode, regno);
3866         do_spill (gen_fr_spill_x, reg, cfa_off, reg);
3867         cfa_off -= 16;
3868       }
3869
3870   gcc_assert (cfa_off == current_frame_info.spill_cfa_off);
3871
3872   finish_spill_pointers ();
3873 }
3874
3875 /* Output the textual info surrounding the prologue.  */
3876
3877 void
3878 ia64_start_function (FILE *file, const char *fnname,
3879                      tree decl ATTRIBUTE_UNUSED)
3880 {
3881 #if TARGET_ABI_OPEN_VMS
3882   vms_start_function (fnname);
3883 #endif
3884
3885   fputs ("\t.proc ", file);
3886   assemble_name (file, fnname);
3887   fputc ('\n', file);
3888   ASM_OUTPUT_LABEL (file, fnname);
3889 }
3890
3891 /* Called after register allocation to add any instructions needed for the
3892    epilogue.  Using an epilogue insn is favored compared to putting all of the
3893    instructions in output_function_prologue(), since it allows the scheduler
3894    to intermix instructions with the saves of the caller saved registers.  In
3895    some cases, it might be necessary to emit a barrier instruction as the last
3896    insn to prevent such scheduling.  */
3897
3898 void
3899 ia64_expand_epilogue (int sibcall_p)
3900 {
3901   rtx_insn *insn;
3902   rtx reg, alt_reg, ar_unat_save_reg;
3903   int regno, alt_regno, cfa_off;
3904
3905   ia64_compute_frame_size (get_frame_size ());
3906
3907   /* If there is a frame pointer, then we use it instead of the stack
3908      pointer, so that the stack pointer does not need to be valid when
3909      the epilogue starts.  See EXIT_IGNORE_STACK.  */
3910   if (frame_pointer_needed)
3911     setup_spill_pointers (current_frame_info.n_spilled,
3912                           hard_frame_pointer_rtx, 0);
3913   else
3914     setup_spill_pointers (current_frame_info.n_spilled, stack_pointer_rtx,
3915                           current_frame_info.total_size);
3916
3917   if (current_frame_info.total_size != 0)
3918     {
3919       /* ??? At this point we must generate a magic insn that appears to
3920          modify the spill iterators and the frame pointer.  This would
3921          allow the most scheduling freedom.  For now, just hard stop.  */
3922       emit_insn (gen_blockage ());
3923     }
3924
3925   /* Locate the bottom of the register save area.  */
3926   cfa_off = (current_frame_info.spill_cfa_off
3927              + current_frame_info.spill_size
3928              + current_frame_info.extra_spill_size);
3929
3930   /* Restore the predicate registers.  */
3931   if (TEST_HARD_REG_BIT (current_frame_info.mask, PR_REG (0)))
3932     {
3933       if (current_frame_info.r[reg_save_pr] != 0)
3934         {
3935           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_pr]);
3936           reg_emitted (reg_save_pr);
3937         }
3938       else
3939         {
3940           alt_regno = next_scratch_gr_reg ();
3941           alt_reg = gen_rtx_REG (DImode, alt_regno);
3942           do_restore (gen_movdi_x, alt_reg, cfa_off);
3943           cfa_off -= 8;
3944         }
3945       reg = gen_rtx_REG (DImode, PR_REG (0));
3946       emit_move_insn (reg, alt_reg);
3947     }
3948
3949   /* Restore the application registers.  */
3950
3951   /* Load the saved unat from the stack, but do not restore it until
3952      after the GRs have been restored.  */
3953   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM))
3954     {
3955       if (current_frame_info.r[reg_save_ar_unat] != 0)
3956         {
3957           ar_unat_save_reg
3958             = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_unat]);
3959           reg_emitted (reg_save_ar_unat);
3960         }
3961       else
3962         {
3963           alt_regno = next_scratch_gr_reg ();
3964           ar_unat_save_reg = gen_rtx_REG (DImode, alt_regno);
3965           current_frame_info.gr_used_mask |= 1 << alt_regno;
3966           do_restore (gen_movdi_x, ar_unat_save_reg, cfa_off);
3967           cfa_off -= 8;
3968         }
3969     }
3970   else
3971     ar_unat_save_reg = NULL_RTX;
3972
3973   if (current_frame_info.r[reg_save_ar_pfs] != 0)
3974     {
3975       reg_emitted (reg_save_ar_pfs);
3976       alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_pfs]);
3977       reg = gen_rtx_REG (DImode, AR_PFS_REGNUM);
3978       emit_move_insn (reg, alt_reg);
3979     }
3980   else if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_PFS_REGNUM))
3981     {
3982       alt_regno = next_scratch_gr_reg ();
3983       alt_reg = gen_rtx_REG (DImode, alt_regno);
3984       do_restore (gen_movdi_x, alt_reg, cfa_off);
3985       cfa_off -= 8;
3986       reg = gen_rtx_REG (DImode, AR_PFS_REGNUM);
3987       emit_move_insn (reg, alt_reg);
3988     }
3989
3990   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_LC_REGNUM))
3991     {
3992       if (current_frame_info.r[reg_save_ar_lc] != 0)
3993         {
3994           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_lc]);
3995           reg_emitted (reg_save_ar_lc);
3996         }
3997       else
3998         {
3999           alt_regno = next_scratch_gr_reg ();
4000           alt_reg = gen_rtx_REG (DImode, alt_regno);
4001           do_restore (gen_movdi_x, alt_reg, cfa_off);
4002           cfa_off -= 8;
4003         }
4004       reg = gen_rtx_REG (DImode, AR_LC_REGNUM);
4005       emit_move_insn (reg, alt_reg);
4006     }
4007
4008   /* Restore the return pointer.  */
4009   if (TEST_HARD_REG_BIT (current_frame_info.mask, BR_REG (0)))
4010     {
4011       if (current_frame_info.r[reg_save_b0] != 0)
4012         {
4013          alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_b0]);
4014          reg_emitted (reg_save_b0);
4015         }
4016       else
4017         {
4018           alt_regno = next_scratch_gr_reg ();
4019           alt_reg = gen_rtx_REG (DImode, alt_regno);
4020           do_restore (gen_movdi_x, alt_reg, cfa_off);
4021           cfa_off -= 8;
4022         }
4023       reg = gen_rtx_REG (DImode, BR_REG (0));
4024       emit_move_insn (reg, alt_reg);
4025     }
4026
4027   /* We should now be at the base of the gr/br/fr spill area.  */
4028   gcc_assert (cfa_off == (current_frame_info.spill_cfa_off
4029                           + current_frame_info.spill_size));
4030
4031   /* The GP may be stored on the stack in the prologue, but it's
4032      never restored in the epilogue.  Skip the stack slot.  */
4033   if (TEST_HARD_REG_BIT (current_frame_info.mask, GR_REG (1)))
4034     cfa_off -= 8;
4035
4036   /* Restore all general registers.  */
4037   for (regno = GR_REG (2); regno <= GR_REG (31); ++regno)
4038     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4039       {
4040         reg = gen_rtx_REG (DImode, regno);
4041         do_restore (gen_gr_restore, reg, cfa_off);
4042         cfa_off -= 8;
4043       }
4044
4045   /* Restore the branch registers.  */
4046   for (regno = BR_REG (1); regno <= BR_REG (7); ++regno)
4047     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4048       {
4049         alt_regno = next_scratch_gr_reg ();
4050         alt_reg = gen_rtx_REG (DImode, alt_regno);
4051         do_restore (gen_movdi_x, alt_reg, cfa_off);
4052         cfa_off -= 8;
4053         reg = gen_rtx_REG (DImode, regno);
4054         emit_move_insn (reg, alt_reg);
4055       }
4056
4057   /* Restore floating point registers.  */
4058   for (regno = FR_REG (2); regno <= FR_REG (127); ++regno)
4059     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4060       {
4061         gcc_assert (!(cfa_off & 15));
4062         reg = gen_rtx_REG (XFmode, regno);
4063         do_restore (gen_fr_restore_x, reg, cfa_off);
4064         cfa_off -= 16;
4065       }
4066
4067   /* Restore ar.unat for real.  */
4068   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM))
4069     {
4070       reg = gen_rtx_REG (DImode, AR_UNAT_REGNUM);
4071       emit_move_insn (reg, ar_unat_save_reg);
4072     }
4073
4074   gcc_assert (cfa_off == current_frame_info.spill_cfa_off);
4075
4076   finish_spill_pointers ();
4077
4078   if (current_frame_info.total_size
4079       || cfun->machine->ia64_eh_epilogue_sp
4080       || frame_pointer_needed)
4081     {
4082       /* ??? At this point we must generate a magic insn that appears to
4083          modify the spill iterators, the stack pointer, and the frame
4084          pointer.  This would allow the most scheduling freedom.  For now,
4085          just hard stop.  */
4086       emit_insn (gen_blockage ());
4087     }
4088
4089   if (cfun->machine->ia64_eh_epilogue_sp)
4090     emit_move_insn (stack_pointer_rtx, cfun->machine->ia64_eh_epilogue_sp);
4091   else if (frame_pointer_needed)
4092     {
4093       insn = emit_move_insn (stack_pointer_rtx, hard_frame_pointer_rtx);
4094       RTX_FRAME_RELATED_P (insn) = 1;
4095       add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);
4096     }
4097   else if (current_frame_info.total_size)
4098     {
4099       rtx offset, frame_size_rtx;
4100
4101       frame_size_rtx = GEN_INT (current_frame_info.total_size);
4102       if (satisfies_constraint_I (frame_size_rtx))
4103         offset = frame_size_rtx;
4104       else
4105         {
4106           regno = next_scratch_gr_reg ();
4107           offset = gen_rtx_REG (DImode, regno);
4108           emit_move_insn (offset, frame_size_rtx);
4109         }
4110
4111       insn = emit_insn (gen_adddi3 (stack_pointer_rtx, stack_pointer_rtx,
4112                                     offset));
4113
4114       RTX_FRAME_RELATED_P (insn) = 1;
4115       add_reg_note (insn, REG_CFA_ADJUST_CFA,
4116                     gen_rtx_SET (stack_pointer_rtx,
4117                                  gen_rtx_PLUS (DImode,
4118                                                stack_pointer_rtx,
4119                                                frame_size_rtx)));
4120     }
4121
4122   if (cfun->machine->ia64_eh_epilogue_bsp)
4123     emit_insn (gen_set_bsp (cfun->machine->ia64_eh_epilogue_bsp));
4124
4125   if (! sibcall_p)
4126     emit_jump_insn (gen_return_internal (gen_rtx_REG (DImode, BR_REG (0))));
4127   else
4128     {
4129       int fp = GR_REG (2);
4130       /* We need a throw away register here, r0 and r1 are reserved,
4131          so r2 is the first available call clobbered register.  If
4132          there was a frame_pointer register, we may have swapped the
4133          names of r2 and HARD_FRAME_POINTER_REGNUM, so we have to make
4134          sure we're using the string "r2" when emitting the register
4135          name for the assembler.  */
4136       if (current_frame_info.r[reg_fp]
4137           && current_frame_info.r[reg_fp] == GR_REG (2))
4138         fp = HARD_FRAME_POINTER_REGNUM;
4139
4140       /* We must emit an alloc to force the input registers to become output
4141          registers.  Otherwise, if the callee tries to pass its parameters
4142          through to another call without an intervening alloc, then these
4143          values get lost.  */
4144       /* ??? We don't need to preserve all input registers.  We only need to
4145          preserve those input registers used as arguments to the sibling call.
4146          It is unclear how to compute that number here.  */
4147       if (current_frame_info.n_input_regs != 0)
4148         {
4149           rtx n_inputs = GEN_INT (current_frame_info.n_input_regs);
4150
4151           insn = emit_insn (gen_alloc (gen_rtx_REG (DImode, fp),
4152                                 const0_rtx, const0_rtx,
4153                                 n_inputs, const0_rtx));
4154           RTX_FRAME_RELATED_P (insn) = 1;
4155
4156           /* ??? We need to mark the alloc as frame-related so that it gets
4157              passed into ia64_asm_unwind_emit for ia64-specific unwinding.
4158              But there's nothing dwarf2 related to be done wrt the register
4159              windows.  If we do nothing, dwarf2out will abort on the UNSPEC;
4160              the empty parallel means dwarf2out will not see anything.  */
4161           add_reg_note (insn, REG_FRAME_RELATED_EXPR,
4162                         gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (0)));
4163         }
4164     }
4165 }
4166
4167 /* Return 1 if br.ret can do all the work required to return from a
4168    function.  */
4169
4170 int
4171 ia64_direct_return (void)
4172 {
4173   if (reload_completed && ! frame_pointer_needed)
4174     {
4175       ia64_compute_frame_size (get_frame_size ());
4176
4177       return (current_frame_info.total_size == 0
4178               && current_frame_info.n_spilled == 0
4179               && current_frame_info.r[reg_save_b0] == 0
4180               && current_frame_info.r[reg_save_pr] == 0
4181               && current_frame_info.r[reg_save_ar_pfs] == 0
4182               && current_frame_info.r[reg_save_ar_unat] == 0
4183               && current_frame_info.r[reg_save_ar_lc] == 0);
4184     }
4185   return 0;
4186 }
4187
4188 /* Return the magic cookie that we use to hold the return address
4189    during early compilation.  */
4190
4191 rtx
4192 ia64_return_addr_rtx (HOST_WIDE_INT count, rtx frame ATTRIBUTE_UNUSED)
4193 {
4194   if (count != 0)
4195     return NULL;
4196   return gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx), UNSPEC_RET_ADDR);
4197 }
4198
4199 /* Split this value after reload, now that we know where the return
4200    address is saved.  */
4201
4202 void
4203 ia64_split_return_addr_rtx (rtx dest)
4204 {
4205   rtx src;
4206
4207   if (TEST_HARD_REG_BIT (current_frame_info.mask, BR_REG (0)))
4208     {
4209       if (current_frame_info.r[reg_save_b0] != 0)
4210         {
4211           src = gen_rtx_REG (DImode, current_frame_info.r[reg_save_b0]);
4212           reg_emitted (reg_save_b0);
4213         }
4214       else
4215         {
4216           HOST_WIDE_INT off;
4217           unsigned int regno;
4218           rtx off_r;
4219
4220           /* Compute offset from CFA for BR0.  */
4221           /* ??? Must be kept in sync with ia64_expand_prologue.  */
4222           off = (current_frame_info.spill_cfa_off
4223                  + current_frame_info.spill_size);
4224           for (regno = GR_REG (1); regno <= GR_REG (31); ++regno)
4225             if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4226               off -= 8;
4227
4228           /* Convert CFA offset to a register based offset.  */
4229           if (frame_pointer_needed)
4230             src = hard_frame_pointer_rtx;
4231           else
4232             {
4233               src = stack_pointer_rtx;
4234               off += current_frame_info.total_size;
4235             }
4236
4237           /* Load address into scratch register.  */
4238           off_r = GEN_INT (off);
4239           if (satisfies_constraint_I (off_r))
4240             emit_insn (gen_adddi3 (dest, src, off_r));
4241           else
4242             {
4243               emit_move_insn (dest, off_r);
4244               emit_insn (gen_adddi3 (dest, src, dest));
4245             }
4246
4247           src = gen_rtx_MEM (Pmode, dest);
4248         }
4249     }
4250   else
4251     src = gen_rtx_REG (DImode, BR_REG (0));
4252
4253   emit_move_insn (dest, src);
4254 }
4255
4256 int
4257 ia64_hard_regno_rename_ok (int from, int to)
4258 {
4259   /* Don't clobber any of the registers we reserved for the prologue.  */
4260   unsigned int r;
4261
4262   for (r = reg_fp; r <= reg_save_ar_lc; r++)
4263     if (to == current_frame_info.r[r]
4264         || from == current_frame_info.r[r]
4265         || to == emitted_frame_related_regs[r]
4266         || from == emitted_frame_related_regs[r])
4267       return 0;
4268
4269   /* Don't use output registers outside the register frame.  */
4270   if (OUT_REGNO_P (to) && to >= OUT_REG (current_frame_info.n_output_regs))
4271     return 0;
4272
4273   /* Retain even/oddness on predicate register pairs.  */
4274   if (PR_REGNO_P (from) && PR_REGNO_P (to))
4275     return (from & 1) == (to & 1);
4276
4277   return 1;
4278 }
4279
4280 /* Implement TARGET_HARD_REGNO_NREGS.
4281
4282    ??? We say that BImode PR values require two registers.  This allows us to
4283    easily store the normal and inverted values.  We use CCImode to indicate
4284    a single predicate register.  */
4285
4286 static unsigned int
4287 ia64_hard_regno_nregs (unsigned int regno, machine_mode mode)
4288 {
4289   if (regno == PR_REG (0) && mode == DImode)
4290     return 64;
4291   if (PR_REGNO_P (regno) && (mode) == BImode)
4292     return 2;
4293   if ((PR_REGNO_P (regno) || GR_REGNO_P (regno)) && mode == CCImode)
4294     return 1;
4295   if (FR_REGNO_P (regno) && mode == XFmode)
4296     return 1;
4297   if (FR_REGNO_P (regno) && mode == RFmode)
4298     return 1;
4299   if (FR_REGNO_P (regno) && mode == XCmode)
4300     return 2;
4301   return CEIL (GET_MODE_SIZE (mode), UNITS_PER_WORD);
4302 }
4303
4304 /* Implement TARGET_HARD_REGNO_MODE_OK.  */
4305
4306 static bool
4307 ia64_hard_regno_mode_ok (unsigned int regno, machine_mode mode)
4308 {
4309   if (FR_REGNO_P (regno))
4310     return (GET_MODE_CLASS (mode) != MODE_CC
4311             && mode != BImode
4312             && mode != TFmode);
4313
4314   if (PR_REGNO_P (regno))
4315     return mode == BImode || GET_MODE_CLASS (mode) == MODE_CC;
4316
4317   if (GR_REGNO_P (regno))
4318     return mode != XFmode && mode != XCmode && mode != RFmode;
4319
4320   if (AR_REGNO_P (regno))
4321     return mode == DImode;
4322
4323   if (BR_REGNO_P (regno))
4324     return mode == DImode;
4325
4326   return false;
4327 }
4328
4329 /* Implement TARGET_MODES_TIEABLE_P.
4330
4331    Don't tie integer and FP modes, as that causes us to get integer registers
4332    allocated for FP instructions.  XFmode only supported in FP registers so
4333    we can't tie it with any other modes.  */
4334
4335 static bool
4336 ia64_modes_tieable_p (machine_mode mode1, machine_mode mode2)
4337 {
4338   return (GET_MODE_CLASS (mode1) == GET_MODE_CLASS (mode2)
4339           && ((mode1 == XFmode || mode1 == XCmode || mode1 == RFmode)
4340               == (mode2 == XFmode || mode2 == XCmode || mode2 == RFmode))
4341           && (mode1 == BImode) == (mode2 == BImode));
4342 }
4343
4344 /* Target hook for assembling integer objects.  Handle word-sized
4345    aligned objects and detect the cases when @fptr is needed.  */
4346
4347 static bool
4348 ia64_assemble_integer (rtx x, unsigned int size, int aligned_p)
4349 {
4350   if (size == POINTER_SIZE / BITS_PER_UNIT
4351       && !(TARGET_NO_PIC || TARGET_AUTO_PIC)
4352       && GET_CODE (x) == SYMBOL_REF
4353       && SYMBOL_REF_FUNCTION_P (x))
4354     {
4355       static const char * const directive[2][2] = {
4356           /* 64-bit pointer */  /* 32-bit pointer */
4357         { "\tdata8.ua\t@fptr(", "\tdata4.ua\t@fptr("},  /* unaligned */
4358         { "\tdata8\t@fptr(",    "\tdata4\t@fptr("}      /* aligned */
4359       };
4360       fputs (directive[(aligned_p != 0)][POINTER_SIZE == 32], asm_out_file);
4361       output_addr_const (asm_out_file, x);
4362       fputs (")\n", asm_out_file);
4363       return true;
4364     }
4365   return default_assemble_integer (x, size, aligned_p);
4366 }
4367
4368 /* Emit the function prologue.  */
4369
4370 static void
4371 ia64_output_function_prologue (FILE *file)
4372 {
4373   int mask, grsave, grsave_prev;
4374
4375   if (current_frame_info.need_regstk)
4376     fprintf (file, "\t.regstk %d, %d, %d, %d\n",
4377              current_frame_info.n_input_regs,
4378              current_frame_info.n_local_regs,
4379              current_frame_info.n_output_regs,
4380              current_frame_info.n_rotate_regs);
4381
4382   if (ia64_except_unwind_info (&global_options) != UI_TARGET)
4383     return;
4384
4385   /* Emit the .prologue directive.  */
4386
4387   mask = 0;
4388   grsave = grsave_prev = 0;
4389   if (current_frame_info.r[reg_save_b0] != 0)
4390     {
4391       mask |= 8;
4392       grsave = grsave_prev = current_frame_info.r[reg_save_b0];
4393     }
4394   if (current_frame_info.r[reg_save_ar_pfs] != 0
4395       && (grsave_prev == 0
4396           || current_frame_info.r[reg_save_ar_pfs] == grsave_prev + 1))
4397     {
4398       mask |= 4;
4399       if (grsave_prev == 0)
4400         grsave = current_frame_info.r[reg_save_ar_pfs];
4401       grsave_prev = current_frame_info.r[reg_save_ar_pfs];
4402     }
4403   if (current_frame_info.r[reg_fp] != 0
4404       && (grsave_prev == 0
4405           || current_frame_info.r[reg_fp] == grsave_prev + 1))
4406     {
4407       mask |= 2;
4408       if (grsave_prev == 0)
4409         grsave = HARD_FRAME_POINTER_REGNUM;
4410       grsave_prev = current_frame_info.r[reg_fp];
4411     }
4412   if (current_frame_info.r[reg_save_pr] != 0
4413       && (grsave_prev == 0
4414           || current_frame_info.r[reg_save_pr] == grsave_prev + 1))
4415     {
4416       mask |= 1;
4417       if (grsave_prev == 0)
4418         grsave = current_frame_info.r[reg_save_pr];
4419     }
4420
4421   if (mask && TARGET_GNU_AS)
4422     fprintf (file, "\t.prologue %d, %d\n", mask,
4423              ia64_dbx_register_number (grsave));
4424   else
4425     fputs ("\t.prologue\n", file);
4426
4427   /* Emit a .spill directive, if necessary, to relocate the base of
4428      the register spill area.  */
4429   if (current_frame_info.spill_cfa_off != -16)
4430     fprintf (file, "\t.spill %ld\n",
4431              (long) (current_frame_info.spill_cfa_off
4432                      + current_frame_info.spill_size));
4433 }
4434
4435 /* Emit the .body directive at the scheduled end of the prologue.  */
4436
4437 static void
4438 ia64_output_function_end_prologue (FILE *file)
4439 {
4440   if (ia64_except_unwind_info (&global_options) != UI_TARGET)
4441     return;
4442
4443   fputs ("\t.body\n", file);
4444 }
4445
4446 /* Emit the function epilogue.  */
4447
4448 static void
4449 ia64_output_function_epilogue (FILE *)
4450 {
4451   int i;
4452
4453   if (current_frame_info.r[reg_fp])
4454     {
4455       const char *tmp = reg_names[HARD_FRAME_POINTER_REGNUM];
4456       reg_names[HARD_FRAME_POINTER_REGNUM]
4457         = reg_names[current_frame_info.r[reg_fp]];
4458       reg_names[current_frame_info.r[reg_fp]] = tmp;
4459       reg_emitted (reg_fp);
4460     }
4461   if (! TARGET_REG_NAMES)
4462     {
4463       for (i = 0; i < current_frame_info.n_input_regs; i++)
4464         reg_names[IN_REG (i)] = ia64_input_reg_names[i];
4465       for (i = 0; i < current_frame_info.n_local_regs; i++)
4466         reg_names[LOC_REG (i)] = ia64_local_reg_names[i];
4467       for (i = 0; i < current_frame_info.n_output_regs; i++)
4468         reg_names[OUT_REG (i)] = ia64_output_reg_names[i];
4469     }
4470
4471   current_frame_info.initialized = 0;
4472 }
4473
4474 int
4475 ia64_dbx_register_number (int regno)
4476 {
4477   /* In ia64_expand_prologue we quite literally renamed the frame pointer
4478      from its home at loc79 to something inside the register frame.  We
4479      must perform the same renumbering here for the debug info.  */
4480   if (current_frame_info.r[reg_fp])
4481     {
4482       if (regno == HARD_FRAME_POINTER_REGNUM)
4483         regno = current_frame_info.r[reg_fp];
4484       else if (regno == current_frame_info.r[reg_fp])
4485         regno = HARD_FRAME_POINTER_REGNUM;
4486     }
4487
4488   if (IN_REGNO_P (regno))
4489     return 32 + regno - IN_REG (0);
4490   else if (LOC_REGNO_P (regno))
4491     return 32 + current_frame_info.n_input_regs + regno - LOC_REG (0);
4492   else if (OUT_REGNO_P (regno))
4493     return (32 + current_frame_info.n_input_regs
4494             + current_frame_info.n_local_regs + regno - OUT_REG (0));
4495   else
4496     return regno;
4497 }
4498
4499 /* Implement TARGET_TRAMPOLINE_INIT.
4500
4501    The trampoline should set the static chain pointer to value placed
4502    into the trampoline and should branch to the specified routine.
4503    To make the normal indirect-subroutine calling convention work,
4504    the trampoline must look like a function descriptor; the first
4505    word being the target address and the second being the target's
4506    global pointer.
4507
4508    We abuse the concept of a global pointer by arranging for it
4509    to point to the data we need to load.  The complete trampoline
4510    has the following form:
4511
4512                 +-------------------+ \
4513         TRAMP:  | __ia64_trampoline | |
4514                 +-------------------+  > fake function descriptor
4515                 | TRAMP+16          | |
4516                 +-------------------+ /
4517                 | target descriptor |
4518                 +-------------------+
4519                 | static link       |
4520                 +-------------------+
4521 */
4522
4523 static void
4524 ia64_trampoline_init (rtx m_tramp, tree fndecl, rtx static_chain)
4525 {
4526   rtx fnaddr = XEXP (DECL_RTL (fndecl), 0);
4527   rtx addr, addr_reg, tramp, eight = GEN_INT (8);
4528
4529   /* The Intel assembler requires that the global __ia64_trampoline symbol
4530      be declared explicitly */
4531   if (!TARGET_GNU_AS)
4532     {
4533       static bool declared_ia64_trampoline = false;
4534
4535       if (!declared_ia64_trampoline)
4536         {
4537           declared_ia64_trampoline = true;
4538           (*targetm.asm_out.globalize_label) (asm_out_file,
4539                                               "__ia64_trampoline");
4540         }
4541     }
4542
4543   /* Make sure addresses are Pmode even if we are in ILP32 mode. */
4544   addr = convert_memory_address (Pmode, XEXP (m_tramp, 0));
4545   fnaddr = convert_memory_address (Pmode, fnaddr);
4546   static_chain = convert_memory_address (Pmode, static_chain);
4547
4548   /* Load up our iterator.  */
4549   addr_reg = copy_to_reg (addr);
4550   m_tramp = adjust_automodify_address (m_tramp, Pmode, addr_reg, 0);
4551
4552   /* The first two words are the fake descriptor:
4553      __ia64_trampoline, ADDR+16.  */
4554   tramp = gen_rtx_SYMBOL_REF (Pmode, "__ia64_trampoline");
4555   if (TARGET_ABI_OPEN_VMS)
4556     {
4557       /* HP decided to break the ELF ABI on VMS (to deal with an ambiguity
4558          in the Macro-32 compiler) and changed the semantics of the LTOFF22
4559          relocation against function symbols to make it identical to the
4560          LTOFF_FPTR22 relocation.  Emit the latter directly to stay within
4561          strict ELF and dereference to get the bare code address.  */
4562       rtx reg = gen_reg_rtx (Pmode);
4563       SYMBOL_REF_FLAGS (tramp) |= SYMBOL_FLAG_FUNCTION;
4564       emit_move_insn (reg, tramp);
4565       emit_move_insn (reg, gen_rtx_MEM (Pmode, reg));
4566       tramp = reg;
4567    }
4568   emit_move_insn (m_tramp, tramp);
4569   emit_insn (gen_adddi3 (addr_reg, addr_reg, eight));
4570   m_tramp = adjust_automodify_address (m_tramp, VOIDmode, NULL, 8);
4571
4572   emit_move_insn (m_tramp, force_reg (Pmode, plus_constant (Pmode, addr, 16)));
4573   emit_insn (gen_adddi3 (addr_reg, addr_reg, eight));
4574   m_tramp = adjust_automodify_address (m_tramp, VOIDmode, NULL, 8);
4575
4576   /* The third word is the target descriptor.  */
4577   emit_move_insn (m_tramp, force_reg (Pmode, fnaddr));
4578   emit_insn (gen_adddi3 (addr_reg, addr_reg, eight));
4579   m_tramp = adjust_automodify_address (m_tramp, VOIDmode, NULL, 8);
4580
4581   /* The fourth word is the static chain.  */
4582   emit_move_insn (m_tramp, static_chain);
4583 }
4584 \f
4585 /* Do any needed setup for a variadic function.  CUM has not been updated
4586    for the last named argument, which is given by ARG.
4587
4588    We generate the actual spill instructions during prologue generation.  */
4589
4590 static void
4591 ia64_setup_incoming_varargs (cumulative_args_t cum,
4592                              const function_arg_info &arg,
4593                              int *pretend_size,
4594                              int second_time ATTRIBUTE_UNUSED)
4595 {
4596   CUMULATIVE_ARGS next_cum = *get_cumulative_args (cum);
4597
4598   /* Skip the current argument.  */
4599   ia64_function_arg_advance (pack_cumulative_args (&next_cum),
4600                              arg.mode, arg.type, arg.named);
4601
4602   if (next_cum.words < MAX_ARGUMENT_SLOTS)
4603     {
4604       int n = MAX_ARGUMENT_SLOTS - next_cum.words;
4605       *pretend_size = n * UNITS_PER_WORD;
4606       cfun->machine->n_varargs = n;
4607     }
4608 }
4609
4610 /* Check whether TYPE is a homogeneous floating point aggregate.  If
4611    it is, return the mode of the floating point type that appears
4612    in all leafs.  If it is not, return VOIDmode.
4613
4614    An aggregate is a homogeneous floating point aggregate is if all
4615    fields/elements in it have the same floating point type (e.g,
4616    SFmode).  128-bit quad-precision floats are excluded.
4617
4618    Variable sized aggregates should never arrive here, since we should
4619    have already decided to pass them by reference.  Top-level zero-sized
4620    aggregates are excluded because our parallels crash the middle-end.  */
4621
4622 static machine_mode
4623 hfa_element_mode (const_tree type, bool nested)
4624 {
4625   machine_mode element_mode = VOIDmode;
4626   machine_mode mode;
4627   enum tree_code code = TREE_CODE (type);
4628   int know_element_mode = 0;
4629   tree t;
4630
4631   if (!nested && (!TYPE_SIZE (type) || integer_zerop (TYPE_SIZE (type))))
4632     return VOIDmode;
4633
4634   switch (code)
4635     {
4636     case VOID_TYPE:     case INTEGER_TYPE:      case ENUMERAL_TYPE:
4637     case BOOLEAN_TYPE:  case POINTER_TYPE:
4638     case OFFSET_TYPE:   case REFERENCE_TYPE:    case METHOD_TYPE:
4639     case LANG_TYPE:             case FUNCTION_TYPE:
4640       return VOIDmode;
4641
4642       /* Fortran complex types are supposed to be HFAs, so we need to handle
4643          gcc's COMPLEX_TYPEs as HFAs.  We need to exclude the integral complex
4644          types though.  */
4645     case COMPLEX_TYPE:
4646       if (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_COMPLEX_FLOAT
4647           && TYPE_MODE (type) != TCmode)
4648         return GET_MODE_INNER (TYPE_MODE (type));
4649       else
4650         return VOIDmode;
4651
4652     case REAL_TYPE:
4653       /* We want to return VOIDmode for raw REAL_TYPEs, but the actual
4654          mode if this is contained within an aggregate.  */
4655       if (nested && TYPE_MODE (type) != TFmode)
4656         return TYPE_MODE (type);
4657       else
4658         return VOIDmode;
4659
4660     case ARRAY_TYPE:
4661       return hfa_element_mode (TREE_TYPE (type), 1);
4662
4663     case RECORD_TYPE:
4664     case UNION_TYPE:
4665     case QUAL_UNION_TYPE:
4666       for (t = TYPE_FIELDS (type); t; t = DECL_CHAIN (t))
4667         {
4668           if (TREE_CODE (t) != FIELD_DECL)
4669             continue;
4670
4671           mode = hfa_element_mode (TREE_TYPE (t), 1);
4672           if (know_element_mode)
4673             {
4674               if (mode != element_mode)
4675                 return VOIDmode;
4676             }
4677           else if (GET_MODE_CLASS (mode) != MODE_FLOAT)
4678             return VOIDmode;
4679           else
4680             {
4681               know_element_mode = 1;
4682               element_mode = mode;
4683             }
4684         }
4685       return element_mode;
4686
4687     default:
4688       /* If we reach here, we probably have some front-end specific type
4689          that the backend doesn't know about.  This can happen via the
4690          aggregate_value_p call in init_function_start.  All we can do is
4691          ignore unknown tree types.  */
4692       return VOIDmode;
4693     }
4694
4695   return VOIDmode;
4696 }
4697
4698 /* Return the number of words required to hold a quantity of TYPE and MODE
4699    when passed as an argument.  */
4700 static int
4701 ia64_function_arg_words (const_tree type, machine_mode mode)
4702 {
4703   int words;
4704
4705   if (mode == BLKmode)
4706     words = int_size_in_bytes (type);
4707   else
4708     words = GET_MODE_SIZE (mode);
4709
4710   return (words + UNITS_PER_WORD - 1) / UNITS_PER_WORD;  /* round up */
4711 }
4712
4713 /* Return the number of registers that should be skipped so the current
4714    argument (described by TYPE and WORDS) will be properly aligned.
4715
4716    Integer and float arguments larger than 8 bytes start at the next
4717    even boundary.  Aggregates larger than 8 bytes start at the next
4718    even boundary if the aggregate has 16 byte alignment.  Note that
4719    in the 32-bit ABI, TImode and TFmode have only 8-byte alignment
4720    but are still to be aligned in registers.
4721
4722    ??? The ABI does not specify how to handle aggregates with
4723    alignment from 9 to 15 bytes, or greater than 16.  We handle them
4724    all as if they had 16 byte alignment.  Such aggregates can occur
4725    only if gcc extensions are used.  */
4726 static int
4727 ia64_function_arg_offset (const CUMULATIVE_ARGS *cum,
4728                           const_tree type, int words)
4729 {
4730   /* No registers are skipped on VMS.  */
4731   if (TARGET_ABI_OPEN_VMS || (cum->words & 1) == 0)
4732     return 0;
4733
4734   if (type
4735       && TREE_CODE (type) != INTEGER_TYPE
4736       && TREE_CODE (type) != REAL_TYPE)
4737     return TYPE_ALIGN (type) > 8 * BITS_PER_UNIT;
4738   else
4739     return words > 1;
4740 }
4741
4742 /* Return rtx for register where argument is passed, or zero if it is passed
4743    on the stack.  */
4744 /* ??? 128-bit quad-precision floats are always passed in general
4745    registers.  */
4746
4747 static rtx
4748 ia64_function_arg_1 (cumulative_args_t cum_v, const function_arg_info &arg,
4749                      bool incoming)
4750 {
4751   const CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
4752
4753   int basereg = (incoming ? GR_ARG_FIRST : AR_ARG_FIRST);
4754   int words = ia64_function_arg_words (arg.type, arg.mode);
4755   int offset = ia64_function_arg_offset (cum, arg.type, words);
4756   machine_mode hfa_mode = VOIDmode;
4757
4758   /* For OPEN VMS, emit the instruction setting up the argument register here,
4759      when we know this will be together with the other arguments setup related
4760      insns.  This is not the conceptually best place to do this, but this is
4761      the easiest as we have convenient access to cumulative args info.  */
4762
4763   if (TARGET_ABI_OPEN_VMS && arg.end_marker_p ())
4764     {
4765       unsigned HOST_WIDE_INT regval = cum->words;
4766       int i;
4767
4768       for (i = 0; i < 8; i++)
4769         regval |= ((int) cum->atypes[i]) << (i * 3 + 8);
4770
4771       emit_move_insn (gen_rtx_REG (DImode, GR_REG (25)),
4772                       GEN_INT (regval));
4773     }
4774
4775   /* If all argument slots are used, then it must go on the stack.  */
4776   if (cum->words + offset >= MAX_ARGUMENT_SLOTS)
4777     return 0;
4778
4779   /* On OpenVMS argument is either in Rn or Fn.  */
4780   if (TARGET_ABI_OPEN_VMS)
4781     {
4782       if (FLOAT_MODE_P (arg.mode))
4783         return gen_rtx_REG (arg.mode, FR_ARG_FIRST + cum->words);
4784       else
4785         return gen_rtx_REG (arg.mode, basereg + cum->words);
4786     }
4787
4788   /* Check for and handle homogeneous FP aggregates.  */
4789   if (arg.type)
4790     hfa_mode = hfa_element_mode (arg.type, 0);
4791
4792   /* Unnamed prototyped hfas are passed as usual.  Named prototyped hfas
4793      and unprototyped hfas are passed specially.  */
4794   if (hfa_mode != VOIDmode && (! cum->prototype || arg.named))
4795     {
4796       rtx loc[16];
4797       int i = 0;
4798       int fp_regs = cum->fp_regs;
4799       int int_regs = cum->words + offset;
4800       int hfa_size = GET_MODE_SIZE (hfa_mode);
4801       int byte_size;
4802       int args_byte_size;
4803
4804       /* If prototyped, pass it in FR regs then GR regs.
4805          If not prototyped, pass it in both FR and GR regs.
4806
4807          If this is an SFmode aggregate, then it is possible to run out of
4808          FR regs while GR regs are still left.  In that case, we pass the
4809          remaining part in the GR regs.  */
4810
4811       /* Fill the FP regs.  We do this always.  We stop if we reach the end
4812          of the argument, the last FP register, or the last argument slot.  */
4813
4814       byte_size = arg.promoted_size_in_bytes ();
4815       args_byte_size = int_regs * UNITS_PER_WORD;
4816       offset = 0;
4817       for (; (offset < byte_size && fp_regs < MAX_ARGUMENT_SLOTS
4818               && args_byte_size < (MAX_ARGUMENT_SLOTS * UNITS_PER_WORD)); i++)
4819         {
4820           loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
4821                                       gen_rtx_REG (hfa_mode, (FR_ARG_FIRST
4822                                                               + fp_regs)),
4823                                       GEN_INT (offset));
4824           offset += hfa_size;
4825           args_byte_size += hfa_size;
4826           fp_regs++;
4827         }
4828
4829       /* If no prototype, then the whole thing must go in GR regs.  */
4830       if (! cum->prototype)
4831         offset = 0;
4832       /* If this is an SFmode aggregate, then we might have some left over
4833          that needs to go in GR regs.  */
4834       else if (byte_size != offset)
4835         int_regs += offset / UNITS_PER_WORD;
4836
4837       /* Fill in the GR regs.  We must use DImode here, not the hfa mode.  */
4838
4839       for (; offset < byte_size && int_regs < MAX_ARGUMENT_SLOTS; i++)
4840         {
4841           machine_mode gr_mode = DImode;
4842           unsigned int gr_size;
4843
4844           /* If we have an odd 4 byte hunk because we ran out of FR regs,
4845              then this goes in a GR reg left adjusted/little endian, right
4846              adjusted/big endian.  */
4847           /* ??? Currently this is handled wrong, because 4-byte hunks are
4848              always right adjusted/little endian.  */
4849           if (offset & 0x4)
4850             gr_mode = SImode;
4851           /* If we have an even 4 byte hunk because the aggregate is a
4852              multiple of 4 bytes in size, then this goes in a GR reg right
4853              adjusted/little endian.  */
4854           else if (byte_size - offset == 4)
4855             gr_mode = SImode;
4856
4857           loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
4858                                       gen_rtx_REG (gr_mode, (basereg
4859                                                              + int_regs)),
4860                                       GEN_INT (offset));
4861
4862           gr_size = GET_MODE_SIZE (gr_mode);
4863           offset += gr_size;
4864           if (gr_size == UNITS_PER_WORD
4865               || (gr_size < UNITS_PER_WORD && offset % UNITS_PER_WORD == 0))
4866             int_regs++;
4867           else if (gr_size > UNITS_PER_WORD)
4868             int_regs += gr_size / UNITS_PER_WORD;
4869         }
4870       return gen_rtx_PARALLEL (arg.mode, gen_rtvec_v (i, loc));
4871     }
4872
4873   /* Integral and aggregates go in general registers.  If we have run out of
4874      FR registers, then FP values must also go in general registers.  This can
4875      happen when we have a SFmode HFA.  */
4876   else if (arg.mode == TFmode || arg.mode == TCmode
4877            || !FLOAT_MODE_P (arg.mode)
4878            || cum->fp_regs == MAX_ARGUMENT_SLOTS)
4879     {
4880       int byte_size = arg.promoted_size_in_bytes ();
4881       if (BYTES_BIG_ENDIAN
4882           && (arg.mode == BLKmode || arg.aggregate_type_p ())
4883           && byte_size < UNITS_PER_WORD
4884           && byte_size > 0)
4885         {
4886           rtx gr_reg = gen_rtx_EXPR_LIST (VOIDmode,
4887                                           gen_rtx_REG (DImode,
4888                                                        (basereg + cum->words
4889                                                         + offset)),
4890                                           const0_rtx);
4891           return gen_rtx_PARALLEL (arg.mode, gen_rtvec (1, gr_reg));
4892         }
4893       else
4894         return gen_rtx_REG (arg.mode, basereg + cum->words + offset);
4895
4896     }
4897
4898   /* If there is a prototype, then FP values go in a FR register when
4899      named, and in a GR register when unnamed.  */
4900   else if (cum->prototype)
4901     {
4902       if (arg.named)
4903         return gen_rtx_REG (arg.mode, FR_ARG_FIRST + cum->fp_regs);
4904       /* In big-endian mode, an anonymous SFmode value must be represented
4905          as (parallel:SF [(expr_list (reg:DI n) (const_int 0))]) to force
4906          the value into the high half of the general register.  */
4907       else if (BYTES_BIG_ENDIAN && arg.mode == SFmode)
4908         return gen_rtx_PARALLEL (arg.mode,
4909                  gen_rtvec (1,
4910                    gen_rtx_EXPR_LIST (VOIDmode,
4911                      gen_rtx_REG (DImode, basereg + cum->words + offset),
4912                                       const0_rtx)));
4913       else
4914         return gen_rtx_REG (arg.mode, basereg + cum->words + offset);
4915     }
4916   /* If there is no prototype, then FP values go in both FR and GR
4917      registers.  */
4918   else
4919     {
4920       /* See comment above.  */
4921       machine_mode inner_mode =
4922         (BYTES_BIG_ENDIAN && arg.mode == SFmode) ? DImode : arg.mode;
4923
4924       rtx fp_reg = gen_rtx_EXPR_LIST (VOIDmode,
4925                                       gen_rtx_REG (arg.mode, (FR_ARG_FIRST
4926                                                           + cum->fp_regs)),
4927                                       const0_rtx);
4928       rtx gr_reg = gen_rtx_EXPR_LIST (VOIDmode,
4929                                       gen_rtx_REG (inner_mode,
4930                                                    (basereg + cum->words
4931                                                     + offset)),
4932                                       const0_rtx);
4933
4934       return gen_rtx_PARALLEL (arg.mode, gen_rtvec (2, fp_reg, gr_reg));
4935     }
4936 }
4937
4938 /* Implement TARGET_FUNCION_ARG target hook.  */
4939
4940 static rtx
4941 ia64_function_arg (cumulative_args_t cum, const function_arg_info &arg)
4942 {
4943   return ia64_function_arg_1 (cum, arg, false);
4944 }
4945
4946 /* Implement TARGET_FUNCION_INCOMING_ARG target hook.  */
4947
4948 static rtx
4949 ia64_function_incoming_arg (cumulative_args_t cum,
4950                             const function_arg_info &arg)
4951 {
4952   return ia64_function_arg_1 (cum, arg, true);
4953 }
4954
4955 /* Return number of bytes, at the beginning of the argument, that must be
4956    put in registers.  0 is the argument is entirely in registers or entirely
4957    in memory.  */
4958
4959 static int
4960 ia64_arg_partial_bytes (cumulative_args_t cum_v, const function_arg_info &arg)
4961 {
4962   CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
4963
4964   int words = ia64_function_arg_words (arg.type, arg.mode);
4965   int offset = ia64_function_arg_offset (cum, arg.type, words);
4966
4967   /* If all argument slots are used, then it must go on the stack.  */
4968   if (cum->words + offset >= MAX_ARGUMENT_SLOTS)
4969     return 0;
4970
4971   /* It doesn't matter whether the argument goes in FR or GR regs.  If
4972      it fits within the 8 argument slots, then it goes entirely in
4973      registers.  If it extends past the last argument slot, then the rest
4974      goes on the stack.  */
4975
4976   if (words + cum->words + offset <= MAX_ARGUMENT_SLOTS)
4977     return 0;
4978
4979   return (MAX_ARGUMENT_SLOTS - cum->words - offset) * UNITS_PER_WORD;
4980 }
4981
4982 /* Return ivms_arg_type based on machine_mode.  */
4983
4984 static enum ivms_arg_type
4985 ia64_arg_type (machine_mode mode)
4986 {
4987   switch (mode)
4988     {
4989     case E_SFmode:
4990       return FS;
4991     case E_DFmode:
4992       return FT;
4993     default:
4994       return I64;
4995     }
4996 }
4997
4998 /* Update CUM to point after this argument.  This is patterned after
4999    ia64_function_arg.  */
5000
5001 static void
5002 ia64_function_arg_advance (cumulative_args_t cum_v, machine_mode mode,
5003                            const_tree type, bool named)
5004 {
5005   CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
5006   int words = ia64_function_arg_words (type, mode);
5007   int offset = ia64_function_arg_offset (cum, type, words);
5008   machine_mode hfa_mode = VOIDmode;
5009
5010   /* If all arg slots are already full, then there is nothing to do.  */
5011   if (cum->words >= MAX_ARGUMENT_SLOTS)
5012     {
5013       cum->words += words + offset;
5014       return;
5015     }
5016
5017   cum->atypes[cum->words] = ia64_arg_type (mode);
5018   cum->words += words + offset;
5019
5020   /* On OpenVMS argument is either in Rn or Fn.  */
5021   if (TARGET_ABI_OPEN_VMS)
5022     {
5023       cum->int_regs = cum->words;
5024       cum->fp_regs = cum->words;
5025       return;
5026     }
5027
5028   /* Check for and handle homogeneous FP aggregates.  */
5029   if (type)
5030     hfa_mode = hfa_element_mode (type, 0);
5031
5032   /* Unnamed prototyped hfas are passed as usual.  Named prototyped hfas
5033      and unprototyped hfas are passed specially.  */
5034   if (hfa_mode != VOIDmode && (! cum->prototype || named))
5035     {
5036       int fp_regs = cum->fp_regs;
5037       /* This is the original value of cum->words + offset.  */
5038       int int_regs = cum->words - words;
5039       int hfa_size = GET_MODE_SIZE (hfa_mode);
5040       int byte_size;
5041       int args_byte_size;
5042
5043       /* If prototyped, pass it in FR regs then GR regs.
5044          If not prototyped, pass it in both FR and GR regs.
5045
5046          If this is an SFmode aggregate, then it is possible to run out of
5047          FR regs while GR regs are still left.  In that case, we pass the
5048          remaining part in the GR regs.  */
5049
5050       /* Fill the FP regs.  We do this always.  We stop if we reach the end
5051          of the argument, the last FP register, or the last argument slot.  */
5052
5053       byte_size = ((mode == BLKmode)
5054                    ? int_size_in_bytes (type) : GET_MODE_SIZE (mode));
5055       args_byte_size = int_regs * UNITS_PER_WORD;
5056       offset = 0;
5057       for (; (offset < byte_size && fp_regs < MAX_ARGUMENT_SLOTS
5058               && args_byte_size < (MAX_ARGUMENT_SLOTS * UNITS_PER_WORD));)
5059         {
5060           offset += hfa_size;
5061           args_byte_size += hfa_size;
5062           fp_regs++;
5063         }
5064
5065       cum->fp_regs = fp_regs;
5066     }
5067
5068   /* Integral and aggregates go in general registers.  So do TFmode FP values.
5069      If we have run out of FR registers, then other FP values must also go in
5070      general registers.  This can happen when we have a SFmode HFA.  */
5071   else if (mode == TFmode || mode == TCmode
5072            || (! FLOAT_MODE_P (mode) || cum->fp_regs == MAX_ARGUMENT_SLOTS))
5073     cum->int_regs = cum->words;
5074
5075   /* If there is a prototype, then FP values go in a FR register when
5076      named, and in a GR register when unnamed.  */
5077   else if (cum->prototype)
5078     {
5079       if (! named)
5080         cum->int_regs = cum->words;
5081       else
5082         /* ??? Complex types should not reach here.  */
5083         cum->fp_regs += (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT ? 2 : 1);
5084     }
5085   /* If there is no prototype, then FP values go in both FR and GR
5086      registers.  */
5087   else
5088     {
5089       /* ??? Complex types should not reach here.  */
5090       cum->fp_regs += (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT ? 2 : 1);
5091       cum->int_regs = cum->words;
5092     }
5093 }
5094
5095 /* Arguments with alignment larger than 8 bytes start at the next even
5096    boundary.  On ILP32 HPUX, TFmode arguments start on next even boundary
5097    even though their normal alignment is 8 bytes.  See ia64_function_arg.  */
5098
5099 static unsigned int
5100 ia64_function_arg_boundary (machine_mode mode, const_tree type)
5101 {
5102   if (mode == TFmode && TARGET_HPUX && TARGET_ILP32)
5103     return PARM_BOUNDARY * 2;
5104
5105   if (type)
5106     {
5107       if (TYPE_ALIGN (type) > PARM_BOUNDARY)
5108         return PARM_BOUNDARY * 2;
5109       else
5110         return PARM_BOUNDARY;
5111     }
5112
5113   if (GET_MODE_BITSIZE (mode) > PARM_BOUNDARY)
5114     return PARM_BOUNDARY * 2;
5115   else
5116     return PARM_BOUNDARY;
5117 }
5118
5119 /* True if it is OK to do sibling call optimization for the specified
5120    call expression EXP.  DECL will be the called function, or NULL if
5121    this is an indirect call.  */
5122 static bool
5123 ia64_function_ok_for_sibcall (tree decl, tree exp ATTRIBUTE_UNUSED)
5124 {
5125   /* We can't perform a sibcall if the current function has the syscall_linkage
5126      attribute.  */
5127   if (lookup_attribute ("syscall_linkage",
5128                         TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
5129     return false;
5130
5131   /* We must always return with our current GP.  This means we can
5132      only sibcall to functions defined in the current module unless
5133      TARGET_CONST_GP is set to true.  */
5134   return (decl && (*targetm.binds_local_p) (decl)) || TARGET_CONST_GP;
5135 }
5136 \f
5137
5138 /* Implement va_arg.  */
5139
5140 static tree
5141 ia64_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
5142                       gimple_seq *post_p)
5143 {
5144   /* Variable sized types are passed by reference.  */
5145   if (pass_va_arg_by_reference (type))
5146     {
5147       tree ptrtype = build_pointer_type (type);
5148       tree addr = std_gimplify_va_arg_expr (valist, ptrtype, pre_p, post_p);
5149       return build_va_arg_indirect_ref (addr);
5150     }
5151
5152   /* Aggregate arguments with alignment larger than 8 bytes start at
5153      the next even boundary.  Integer and floating point arguments
5154      do so if they are larger than 8 bytes, whether or not they are
5155      also aligned larger than 8 bytes.  */
5156   if ((TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == INTEGER_TYPE)
5157       ? int_size_in_bytes (type) > 8 : TYPE_ALIGN (type) > 8 * BITS_PER_UNIT)
5158     {
5159       tree t = fold_build_pointer_plus_hwi (valist, 2 * UNITS_PER_WORD - 1);
5160       t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
5161                   build_int_cst (TREE_TYPE (t), -2 * UNITS_PER_WORD));
5162       gimplify_assign (unshare_expr (valist), t, pre_p);
5163     }
5164
5165   return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
5166 }
5167 \f
5168 /* Return 1 if function return value returned in memory.  Return 0 if it is
5169    in a register.  */
5170
5171 static bool
5172 ia64_return_in_memory (const_tree valtype, const_tree fntype ATTRIBUTE_UNUSED)
5173 {
5174   machine_mode mode;
5175   machine_mode hfa_mode;
5176   HOST_WIDE_INT byte_size;
5177
5178   mode = TYPE_MODE (valtype);
5179   byte_size = GET_MODE_SIZE (mode);
5180   if (mode == BLKmode)
5181     {
5182       byte_size = int_size_in_bytes (valtype);
5183       if (byte_size < 0)
5184         return true;
5185     }
5186
5187   /* Hfa's with up to 8 elements are returned in the FP argument registers.  */
5188
5189   hfa_mode = hfa_element_mode (valtype, 0);
5190   if (hfa_mode != VOIDmode)
5191     {
5192       int hfa_size = GET_MODE_SIZE (hfa_mode);
5193
5194       if (byte_size / hfa_size > MAX_ARGUMENT_SLOTS)
5195         return true;
5196       else
5197         return false;
5198     }
5199   else if (byte_size > UNITS_PER_WORD * MAX_INT_RETURN_SLOTS)
5200     return true;
5201   else
5202     return false;
5203 }
5204
5205 /* Return rtx for register that holds the function return value.  */
5206
5207 static rtx
5208 ia64_function_value (const_tree valtype,
5209                      const_tree fn_decl_or_type,
5210                      bool outgoing ATTRIBUTE_UNUSED)
5211 {
5212   machine_mode mode;
5213   machine_mode hfa_mode;
5214   int unsignedp;
5215   const_tree func = fn_decl_or_type;
5216
5217   if (fn_decl_or_type
5218       && !DECL_P (fn_decl_or_type))
5219     func = NULL;
5220
5221   mode = TYPE_MODE (valtype);
5222   hfa_mode = hfa_element_mode (valtype, 0);
5223
5224   if (hfa_mode != VOIDmode)
5225     {
5226       rtx loc[8];
5227       int i;
5228       int hfa_size;
5229       int byte_size;
5230       int offset;
5231
5232       hfa_size = GET_MODE_SIZE (hfa_mode);
5233       byte_size = ((mode == BLKmode)
5234                    ? int_size_in_bytes (valtype) : GET_MODE_SIZE (mode));
5235       offset = 0;
5236       for (i = 0; offset < byte_size; i++)
5237         {
5238           loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
5239                                       gen_rtx_REG (hfa_mode, FR_ARG_FIRST + i),
5240                                       GEN_INT (offset));
5241           offset += hfa_size;
5242         }
5243       return gen_rtx_PARALLEL (mode, gen_rtvec_v (i, loc));
5244     }
5245   else if (FLOAT_TYPE_P (valtype) && mode != TFmode && mode != TCmode)
5246     return gen_rtx_REG (mode, FR_ARG_FIRST);
5247   else
5248     {
5249       bool need_parallel = false;
5250
5251       /* In big-endian mode, we need to manage the layout of aggregates
5252          in the registers so that we get the bits properly aligned in
5253          the highpart of the registers.  */
5254       if (BYTES_BIG_ENDIAN
5255           && (mode == BLKmode || (valtype && AGGREGATE_TYPE_P (valtype))))
5256         need_parallel = true;
5257
5258       /* Something like struct S { long double x; char a[0] } is not an
5259          HFA structure, and therefore doesn't go in fp registers.  But
5260          the middle-end will give it XFmode anyway, and XFmode values
5261          don't normally fit in integer registers.  So we need to smuggle
5262          the value inside a parallel.  */
5263       else if (mode == XFmode || mode == XCmode || mode == RFmode)
5264         need_parallel = true;
5265
5266       if (need_parallel)
5267         {
5268           rtx loc[8];
5269           int offset;
5270           int bytesize;
5271           int i;
5272
5273           offset = 0;
5274           bytesize = int_size_in_bytes (valtype);
5275           /* An empty PARALLEL is invalid here, but the return value
5276              doesn't matter for empty structs.  */
5277           if (bytesize == 0)
5278             return gen_rtx_REG (mode, GR_RET_FIRST);
5279           for (i = 0; offset < bytesize; i++)
5280             {
5281               loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
5282                                           gen_rtx_REG (DImode,
5283                                                        GR_RET_FIRST + i),
5284                                           GEN_INT (offset));
5285               offset += UNITS_PER_WORD;
5286             }
5287           return gen_rtx_PARALLEL (mode, gen_rtvec_v (i, loc));
5288         }
5289
5290       mode = promote_function_mode (valtype, mode, &unsignedp,
5291                                     func ? TREE_TYPE (func) : NULL_TREE,
5292                                     true);
5293
5294       return gen_rtx_REG (mode, GR_RET_FIRST);
5295     }
5296 }
5297
5298 /* Worker function for TARGET_LIBCALL_VALUE.  */
5299
5300 static rtx
5301 ia64_libcall_value (machine_mode mode,
5302                     const_rtx fun ATTRIBUTE_UNUSED)
5303 {
5304   return gen_rtx_REG (mode,
5305                       (((GET_MODE_CLASS (mode) == MODE_FLOAT
5306                          || GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT)
5307                         && (mode) != TFmode)
5308                        ? FR_RET_FIRST : GR_RET_FIRST));
5309 }
5310
5311 /* Worker function for FUNCTION_VALUE_REGNO_P.  */
5312
5313 static bool
5314 ia64_function_value_regno_p (const unsigned int regno)
5315 {
5316   return ((regno >= GR_RET_FIRST && regno <= GR_RET_LAST)
5317           || (regno >= FR_RET_FIRST && regno <= FR_RET_LAST));
5318 }
5319
5320 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
5321    We need to emit DTP-relative relocations.  */
5322
5323 static void
5324 ia64_output_dwarf_dtprel (FILE *file, int size, rtx x)
5325 {
5326   gcc_assert (size == 4 || size == 8);
5327   if (size == 4)
5328     fputs ("\tdata4.ua\t@dtprel(", file);
5329   else
5330     fputs ("\tdata8.ua\t@dtprel(", file);
5331   output_addr_const (file, x);
5332   fputs (")", file);
5333 }
5334
5335 /* Print a memory address as an operand to reference that memory location.  */
5336
5337 /* ??? Do we need this?  It gets used only for 'a' operands.  We could perhaps
5338    also call this from ia64_print_operand for memory addresses.  */
5339
5340 static void
5341 ia64_print_operand_address (FILE * stream ATTRIBUTE_UNUSED,
5342                             machine_mode /*mode*/,
5343                             rtx address ATTRIBUTE_UNUSED)
5344 {
5345 }
5346
5347 /* Print an operand to an assembler instruction.
5348    C    Swap and print a comparison operator.
5349    D    Print an FP comparison operator.
5350    E    Print 32 - constant, for SImode shifts as extract.
5351    e    Print 64 - constant, for DImode rotates.
5352    F    A floating point constant 0.0 emitted as f0, or 1.0 emitted as f1, or
5353         a floating point register emitted normally.
5354    G    A floating point constant.
5355    I    Invert a predicate register by adding 1.
5356    J    Select the proper predicate register for a condition.
5357    j    Select the inverse predicate register for a condition.
5358    O    Append .acq for volatile load.
5359    P    Postincrement of a MEM.
5360    Q    Append .rel for volatile store.
5361    R    Print .s .d or nothing for a single, double or no truncation.
5362    S    Shift amount for shladd instruction.
5363    T    Print an 8-bit sign extended number (K) as a 32-bit unsigned number
5364         for Intel assembler.
5365    U    Print an 8-bit sign extended number (K) as a 64-bit unsigned number
5366         for Intel assembler.
5367    X    A pair of floating point registers.
5368    r    Print register name, or constant 0 as r0.  HP compatibility for
5369         Linux kernel.
5370    v    Print vector constant value as an 8-byte integer value.  */
5371
5372 static void
5373 ia64_print_operand (FILE * file, rtx x, int code)
5374 {
5375   const char *str;
5376
5377   switch (code)
5378     {
5379     case 0:
5380       /* Handled below.  */
5381       break;
5382
5383     case 'C':
5384       {
5385         enum rtx_code c = swap_condition (GET_CODE (x));
5386         fputs (GET_RTX_NAME (c), file);
5387         return;
5388       }
5389
5390     case 'D':
5391       switch (GET_CODE (x))
5392         {
5393         case NE:
5394           str = "neq";
5395           break;
5396         case UNORDERED:
5397           str = "unord";
5398           break;
5399         case ORDERED:
5400           str = "ord";
5401           break;
5402         case UNLT:
5403           str = "nge";
5404           break;
5405         case UNLE:
5406           str = "ngt";
5407           break;
5408         case UNGT:
5409           str = "nle";
5410           break;
5411         case UNGE:
5412           str = "nlt";
5413           break;
5414         case UNEQ:
5415         case LTGT:
5416           gcc_unreachable ();
5417         default:
5418           str = GET_RTX_NAME (GET_CODE (x));
5419           break;
5420         }
5421       fputs (str, file);
5422       return;
5423
5424     case 'E':
5425       fprintf (file, HOST_WIDE_INT_PRINT_DEC, 32 - INTVAL (x));
5426       return;
5427
5428     case 'e':
5429       fprintf (file, HOST_WIDE_INT_PRINT_DEC, 64 - INTVAL (x));
5430       return;
5431
5432     case 'F':
5433       if (x == CONST0_RTX (GET_MODE (x)))
5434         str = reg_names [FR_REG (0)];
5435       else if (x == CONST1_RTX (GET_MODE (x)))
5436         str = reg_names [FR_REG (1)];
5437       else
5438         {
5439           gcc_assert (GET_CODE (x) == REG);
5440           str = reg_names [REGNO (x)];
5441         }
5442       fputs (str, file);
5443       return;
5444
5445     case 'G':
5446       {
5447         long val[4];
5448         real_to_target (val, CONST_DOUBLE_REAL_VALUE (x), GET_MODE (x));
5449         if (GET_MODE (x) == SFmode)
5450           fprintf (file, "0x%08lx", val[0] & 0xffffffff);
5451         else if (GET_MODE (x) == DFmode)
5452           fprintf (file, "0x%08lx%08lx", (WORDS_BIG_ENDIAN ? val[0] : val[1])
5453                                           & 0xffffffff,
5454                                          (WORDS_BIG_ENDIAN ? val[1] : val[0])
5455                                           & 0xffffffff);
5456         else
5457           output_operand_lossage ("invalid %%G mode");
5458       }
5459       return;
5460
5461     case 'I':
5462       fputs (reg_names [REGNO (x) + 1], file);
5463       return;
5464
5465     case 'J':
5466     case 'j':
5467       {
5468         unsigned int regno = REGNO (XEXP (x, 0));
5469         if (GET_CODE (x) == EQ)
5470           regno += 1;
5471         if (code == 'j')
5472           regno ^= 1;
5473         fputs (reg_names [regno], file);
5474       }
5475       return;
5476
5477     case 'O':
5478       if (MEM_VOLATILE_P (x))
5479         fputs(".acq", file);
5480       return;
5481
5482     case 'P':
5483       {
5484         HOST_WIDE_INT value;
5485
5486         switch (GET_CODE (XEXP (x, 0)))
5487           {
5488           default:
5489             return;
5490
5491           case POST_MODIFY:
5492             x = XEXP (XEXP (XEXP (x, 0), 1), 1);
5493             if (GET_CODE (x) == CONST_INT)
5494               value = INTVAL (x);
5495             else
5496               {
5497                 gcc_assert (GET_CODE (x) == REG);
5498                 fprintf (file, ", %s", reg_names[REGNO (x)]);
5499                 return;
5500               }
5501             break;
5502
5503           case POST_INC:
5504             value = GET_MODE_SIZE (GET_MODE (x));
5505             break;
5506
5507           case POST_DEC:
5508             value = - (HOST_WIDE_INT) GET_MODE_SIZE (GET_MODE (x));
5509             break;
5510           }
5511
5512         fprintf (file, ", " HOST_WIDE_INT_PRINT_DEC, value);
5513         return;
5514       }
5515
5516     case 'Q':
5517       if (MEM_VOLATILE_P (x))
5518         fputs(".rel", file);
5519       return;
5520
5521     case 'R':
5522       if (x == CONST0_RTX (GET_MODE (x)))
5523         fputs(".s", file);
5524       else if (x == CONST1_RTX (GET_MODE (x)))
5525         fputs(".d", file);
5526       else if (x == CONST2_RTX (GET_MODE (x)))
5527         ;
5528       else
5529         output_operand_lossage ("invalid %%R value");
5530       return;
5531
5532     case 'S':
5533       fprintf (file, "%d", exact_log2 (INTVAL (x)));
5534       return;
5535
5536     case 'T':
5537       if (! TARGET_GNU_AS && GET_CODE (x) == CONST_INT)
5538         {
5539           fprintf (file, "0x%x", (int) INTVAL (x) & 0xffffffff);
5540           return;
5541         }
5542       break;
5543
5544     case 'U':
5545       if (! TARGET_GNU_AS && GET_CODE (x) == CONST_INT)
5546         {
5547           const char *prefix = "0x";
5548           if (INTVAL (x) & 0x80000000)
5549             {
5550               fprintf (file, "0xffffffff");
5551               prefix = "";
5552             }
5553           fprintf (file, "%s%x", prefix, (int) INTVAL (x) & 0xffffffff);
5554           return;
5555         }
5556       break;
5557
5558     case 'X':
5559       {
5560         unsigned int regno = REGNO (x);
5561         fprintf (file, "%s, %s", reg_names [regno], reg_names [regno + 1]);
5562       }
5563       return;
5564
5565     case 'r':
5566       /* If this operand is the constant zero, write it as register zero.
5567          Any register, zero, or CONST_INT value is OK here.  */
5568       if (GET_CODE (x) == REG)
5569         fputs (reg_names[REGNO (x)], file);
5570       else if (x == CONST0_RTX (GET_MODE (x)))
5571         fputs ("r0", file);
5572       else if (GET_CODE (x) == CONST_INT)
5573         output_addr_const (file, x);
5574       else
5575         output_operand_lossage ("invalid %%r value");
5576       return;
5577
5578     case 'v':
5579       gcc_assert (GET_CODE (x) == CONST_VECTOR);
5580       x = simplify_subreg (DImode, x, GET_MODE (x), 0);
5581       break;
5582
5583     case '+':
5584       {
5585         const char *which;
5586
5587         /* For conditional branches, returns or calls, substitute
5588            sptk, dptk, dpnt, or spnt for %s.  */
5589         x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
5590         if (x)
5591           {
5592             int pred_val = profile_probability::from_reg_br_prob_note
5593                                  (XINT (x, 0)).to_reg_br_prob_base ();
5594
5595             /* Guess top and bottom 10% statically predicted.  */
5596             if (pred_val < REG_BR_PROB_BASE / 50
5597                 && br_prob_note_reliable_p (x))
5598               which = ".spnt";
5599             else if (pred_val < REG_BR_PROB_BASE / 2)
5600               which = ".dpnt";
5601             else if (pred_val < REG_BR_PROB_BASE / 100 * 98
5602                      || !br_prob_note_reliable_p (x))
5603               which = ".dptk";
5604             else
5605               which = ".sptk";
5606           }
5607         else if (CALL_P (current_output_insn))
5608           which = ".sptk";
5609         else
5610           which = ".dptk";
5611
5612         fputs (which, file);
5613         return;
5614       }
5615
5616     case ',':
5617       x = current_insn_predicate;
5618       if (x)
5619         {
5620           unsigned int regno = REGNO (XEXP (x, 0));
5621           if (GET_CODE (x) == EQ)
5622             regno += 1;
5623           fprintf (file, "(%s) ", reg_names [regno]);
5624         }
5625       return;
5626
5627     default:
5628       output_operand_lossage ("ia64_print_operand: unknown code");
5629       return;
5630     }
5631
5632   switch (GET_CODE (x))
5633     {
5634       /* This happens for the spill/restore instructions.  */
5635     case POST_INC:
5636     case POST_DEC:
5637     case POST_MODIFY:
5638       x = XEXP (x, 0);
5639       /* fall through */
5640
5641     case REG:
5642       fputs (reg_names [REGNO (x)], file);
5643       break;
5644
5645     case MEM:
5646       {
5647         rtx addr = XEXP (x, 0);
5648         if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
5649           addr = XEXP (addr, 0);
5650         fprintf (file, "[%s]", reg_names [REGNO (addr)]);
5651         break;
5652       }
5653
5654     default:
5655       output_addr_const (file, x);
5656       break;
5657     }
5658
5659   return;
5660 }
5661
5662 /* Worker function for TARGET_PRINT_OPERAND_PUNCT_VALID_P.  */
5663
5664 static bool
5665 ia64_print_operand_punct_valid_p (unsigned char code)
5666 {
5667   return (code == '+' || code == ',');
5668 }
5669 \f
5670 /* Compute a (partial) cost for rtx X.  Return true if the complete
5671    cost has been computed, and false if subexpressions should be
5672    scanned.  In either case, *TOTAL contains the cost result.  */
5673 /* ??? This is incomplete.  */
5674
5675 static bool
5676 ia64_rtx_costs (rtx x, machine_mode mode, int outer_code,
5677                 int opno ATTRIBUTE_UNUSED,
5678                 int *total, bool speed ATTRIBUTE_UNUSED)
5679 {
5680   int code = GET_CODE (x);
5681
5682   switch (code)
5683     {
5684     case CONST_INT:
5685       switch (outer_code)
5686         {
5687         case SET:
5688           *total = satisfies_constraint_J (x) ? 0 : COSTS_N_INSNS (1);
5689           return true;
5690         case PLUS:
5691           if (satisfies_constraint_I (x))
5692             *total = 0;
5693           else if (satisfies_constraint_J (x))
5694             *total = 1;
5695           else
5696             *total = COSTS_N_INSNS (1);
5697           return true;
5698         default:
5699           if (satisfies_constraint_K (x) || satisfies_constraint_L (x))
5700             *total = 0;
5701           else
5702             *total = COSTS_N_INSNS (1);
5703           return true;
5704         }
5705
5706     case CONST_DOUBLE:
5707       *total = COSTS_N_INSNS (1);
5708       return true;
5709
5710     case CONST:
5711     case SYMBOL_REF:
5712     case LABEL_REF:
5713       *total = COSTS_N_INSNS (3);
5714       return true;
5715
5716     case FMA:
5717       *total = COSTS_N_INSNS (4);
5718       return true;
5719
5720     case MULT:
5721       /* For multiplies wider than HImode, we have to go to the FPU,
5722          which normally involves copies.  Plus there's the latency
5723          of the multiply itself, and the latency of the instructions to
5724          transfer integer regs to FP regs.  */
5725       if (FLOAT_MODE_P (mode))
5726         *total = COSTS_N_INSNS (4);
5727       else if (GET_MODE_SIZE (mode) > 2)
5728         *total = COSTS_N_INSNS (10);
5729       else
5730         *total = COSTS_N_INSNS (2);
5731       return true;
5732
5733     case PLUS:
5734     case MINUS:
5735       if (FLOAT_MODE_P (mode))
5736         {
5737           *total = COSTS_N_INSNS (4);
5738           return true;
5739         }
5740       /* FALLTHRU */
5741
5742     case ASHIFT:
5743     case ASHIFTRT:
5744     case LSHIFTRT:
5745       *total = COSTS_N_INSNS (1);
5746       return true;
5747
5748     case DIV:
5749     case UDIV:
5750     case MOD:
5751     case UMOD:
5752       /* We make divide expensive, so that divide-by-constant will be
5753          optimized to a multiply.  */
5754       *total = COSTS_N_INSNS (60);
5755       return true;
5756
5757     default:
5758       return false;
5759     }
5760 }
5761
5762 /* Calculate the cost of moving data from a register in class FROM to
5763    one in class TO, using MODE.  */
5764
5765 static int
5766 ia64_register_move_cost (machine_mode mode, reg_class_t from,
5767                          reg_class_t to)
5768 {
5769   /* ADDL_REGS is the same as GR_REGS for movement purposes.  */
5770   if (to == ADDL_REGS)
5771     to = GR_REGS;
5772   if (from == ADDL_REGS)
5773     from = GR_REGS;
5774
5775   /* All costs are symmetric, so reduce cases by putting the
5776      lower number class as the destination.  */
5777   if (from < to)
5778     {
5779       reg_class_t tmp = to;
5780       to = from, from = tmp;
5781     }
5782
5783   /* Moving from FR<->GR in XFmode must be more expensive than 2,
5784      so that we get secondary memory reloads.  Between FR_REGS,
5785      we have to make this at least as expensive as memory_move_cost
5786      to avoid spectacularly poor register class preferencing.  */
5787   if (mode == XFmode || mode == RFmode)
5788     {
5789       if (to != GR_REGS || from != GR_REGS)
5790         return memory_move_cost (mode, to, false);
5791       else
5792         return 3;
5793     }
5794
5795   switch (to)
5796     {
5797     case PR_REGS:
5798       /* Moving between PR registers takes two insns.  */
5799       if (from == PR_REGS)
5800         return 3;
5801       /* Moving between PR and anything but GR is impossible.  */
5802       if (from != GR_REGS)
5803         return memory_move_cost (mode, to, false);
5804       break;
5805
5806     case BR_REGS:
5807       /* Moving between BR and anything but GR is impossible.  */
5808       if (from != GR_REGS && from != GR_AND_BR_REGS)
5809         return memory_move_cost (mode, to, false);
5810       break;
5811
5812     case AR_I_REGS:
5813     case AR_M_REGS:
5814       /* Moving between AR and anything but GR is impossible.  */
5815       if (from != GR_REGS)
5816         return memory_move_cost (mode, to, false);
5817       break;
5818
5819     case GR_REGS:
5820     case FR_REGS:
5821     case FP_REGS:
5822     case GR_AND_FR_REGS:
5823     case GR_AND_BR_REGS:
5824     case ALL_REGS:
5825       break;
5826
5827     default:
5828       gcc_unreachable ();
5829     }
5830
5831   return 2;
5832 }
5833
5834 /* Calculate the cost of moving data of MODE from a register to or from
5835    memory.  */
5836
5837 static int
5838 ia64_memory_move_cost (machine_mode mode ATTRIBUTE_UNUSED,
5839                        reg_class_t rclass,
5840                        bool in ATTRIBUTE_UNUSED)
5841 {
5842   if (rclass == GENERAL_REGS
5843       || rclass == FR_REGS
5844       || rclass == FP_REGS
5845       || rclass == GR_AND_FR_REGS)
5846     return 4;
5847   else
5848     return 10;
5849 }
5850
5851 /* Implement TARGET_PREFERRED_RELOAD_CLASS.  Place additional restrictions
5852    on RCLASS to use when copying X into that class.  */
5853
5854 static reg_class_t
5855 ia64_preferred_reload_class (rtx x, reg_class_t rclass)
5856 {
5857   switch (rclass)
5858     {
5859     case FR_REGS:
5860     case FP_REGS:
5861       /* Don't allow volatile mem reloads into floating point registers.
5862          This is defined to force reload to choose the r/m case instead
5863          of the f/f case when reloading (set (reg fX) (mem/v)).  */
5864       if (MEM_P (x) && MEM_VOLATILE_P (x))
5865         return NO_REGS;
5866
5867       /* Force all unrecognized constants into the constant pool.  */
5868       if (CONSTANT_P (x))
5869         return NO_REGS;
5870       break;
5871
5872     case AR_M_REGS:
5873     case AR_I_REGS:
5874       if (!OBJECT_P (x))
5875         return NO_REGS;
5876       break;
5877
5878     default:
5879       break;
5880     }
5881
5882   return rclass;
5883 }
5884
5885 /* This function returns the register class required for a secondary
5886    register when copying between one of the registers in RCLASS, and X,
5887    using MODE.  A return value of NO_REGS means that no secondary register
5888    is required.  */
5889
5890 enum reg_class
5891 ia64_secondary_reload_class (enum reg_class rclass,
5892                              machine_mode mode ATTRIBUTE_UNUSED, rtx x)
5893 {
5894   int regno = -1;
5895
5896   if (GET_CODE (x) == REG || GET_CODE (x) == SUBREG)
5897     regno = true_regnum (x);
5898
5899   switch (rclass)
5900     {
5901     case BR_REGS:
5902     case AR_M_REGS:
5903     case AR_I_REGS:
5904       /* ??? BR<->BR register copies can happen due to a bad gcse/cse/global
5905          interaction.  We end up with two pseudos with overlapping lifetimes
5906          both of which are equiv to the same constant, and both which need
5907          to be in BR_REGS.  This seems to be a cse bug.  cse_basic_block_end
5908          changes depending on the path length, which means the qty_first_reg
5909          check in make_regs_eqv can give different answers at different times.
5910          At some point I'll probably need a reload_indi pattern to handle
5911          this.
5912
5913          We can also get GR_AND_FR_REGS to BR_REGS/AR_REGS copies, where we
5914          wound up with a FP register from GR_AND_FR_REGS.  Extend that to all
5915          non-general registers for good measure.  */
5916       if (regno >= 0 && ! GENERAL_REGNO_P (regno))
5917         return GR_REGS;
5918
5919       /* This is needed if a pseudo used as a call_operand gets spilled to a
5920          stack slot.  */
5921       if (GET_CODE (x) == MEM)
5922         return GR_REGS;
5923       break;
5924
5925     case FR_REGS:
5926     case FP_REGS:
5927       /* Need to go through general registers to get to other class regs.  */
5928       if (regno >= 0 && ! (FR_REGNO_P (regno) || GENERAL_REGNO_P (regno)))
5929         return GR_REGS;
5930
5931       /* This can happen when a paradoxical subreg is an operand to the
5932          muldi3 pattern.  */
5933       /* ??? This shouldn't be necessary after instruction scheduling is
5934          enabled, because paradoxical subregs are not accepted by
5935          register_operand when INSN_SCHEDULING is defined.  Or alternatively,
5936          stop the paradoxical subreg stupidity in the *_operand functions
5937          in recog.c.  */
5938       if (GET_CODE (x) == MEM
5939           && (GET_MODE (x) == SImode || GET_MODE (x) == HImode
5940               || GET_MODE (x) == QImode))
5941         return GR_REGS;
5942
5943       /* This can happen because of the ior/and/etc patterns that accept FP
5944          registers as operands.  If the third operand is a constant, then it
5945          needs to be reloaded into a FP register.  */
5946       if (GET_CODE (x) == CONST_INT)
5947         return GR_REGS;
5948
5949       /* This can happen because of register elimination in a muldi3 insn.
5950          E.g. `26107 * (unsigned long)&u'.  */
5951       if (GET_CODE (x) == PLUS)
5952         return GR_REGS;
5953       break;
5954
5955     case PR_REGS:
5956       /* ??? This happens if we cse/gcse a BImode value across a call,
5957          and the function has a nonlocal goto.  This is because global
5958          does not allocate call crossing pseudos to hard registers when
5959          crtl->has_nonlocal_goto is true.  This is relatively
5960          common for C++ programs that use exceptions.  To reproduce,
5961          return NO_REGS and compile libstdc++.  */
5962       if (GET_CODE (x) == MEM)
5963         return GR_REGS;
5964
5965       /* This can happen when we take a BImode subreg of a DImode value,
5966          and that DImode value winds up in some non-GR register.  */
5967       if (regno >= 0 && ! GENERAL_REGNO_P (regno) && ! PR_REGNO_P (regno))
5968         return GR_REGS;
5969       break;
5970
5971     default:
5972       break;
5973     }
5974
5975   return NO_REGS;
5976 }
5977
5978 \f
5979 /* Implement targetm.unspec_may_trap_p hook.  */
5980 static int
5981 ia64_unspec_may_trap_p (const_rtx x, unsigned flags)
5982 {
5983   switch (XINT (x, 1))
5984     {
5985     case UNSPEC_LDA:
5986     case UNSPEC_LDS:
5987     case UNSPEC_LDSA:
5988     case UNSPEC_LDCCLR:
5989     case UNSPEC_CHKACLR:
5990     case UNSPEC_CHKS:
5991       /* These unspecs are just wrappers.  */
5992       return may_trap_p_1 (XVECEXP (x, 0, 0), flags);
5993     }
5994
5995   return default_unspec_may_trap_p (x, flags);
5996 }
5997
5998 \f
5999 /* Parse the -mfixed-range= option string.  */
6000
6001 static void
6002 fix_range (const char *const_str)
6003 {
6004   int i, first, last;
6005   char *str, *dash, *comma;
6006
6007   /* str must be of the form REG1'-'REG2{,REG1'-'REG} where REG1 and
6008      REG2 are either register names or register numbers.  The effect
6009      of this option is to mark the registers in the range from REG1 to
6010      REG2 as ``fixed'' so they won't be used by the compiler.  This is
6011      used, e.g., to ensure that kernel mode code doesn't use f32-f127.  */
6012
6013   i = strlen (const_str);
6014   str = (char *) alloca (i + 1);
6015   memcpy (str, const_str, i + 1);
6016
6017   while (1)
6018     {
6019       dash = strchr (str, '-');
6020       if (!dash)
6021         {
6022           warning (0, "value of %<-mfixed-range%> must have form REG1-REG2");
6023           return;
6024         }
6025       *dash = '\0';
6026
6027       comma = strchr (dash + 1, ',');
6028       if (comma)
6029         *comma = '\0';
6030
6031       first = decode_reg_name (str);
6032       if (first < 0)
6033         {
6034           warning (0, "unknown register name: %s", str);
6035           return;
6036         }
6037
6038       last = decode_reg_name (dash + 1);
6039       if (last < 0)
6040         {
6041           warning (0, "unknown register name: %s", dash + 1);
6042           return;
6043         }
6044
6045       *dash = '-';
6046
6047       if (first > last)
6048         {
6049           warning (0, "%s-%s is an empty range", str, dash + 1);
6050           return;
6051         }
6052
6053       for (i = first; i <= last; ++i)
6054         fixed_regs[i] = call_used_regs[i] = 1;
6055
6056       if (!comma)
6057         break;
6058
6059       *comma = ',';
6060       str = comma + 1;
6061     }
6062 }
6063
6064 /* Implement TARGET_OPTION_OVERRIDE.  */
6065
6066 static void
6067 ia64_option_override (void)
6068 {
6069   unsigned int i;
6070   cl_deferred_option *opt;
6071   vec<cl_deferred_option> *v
6072     = (vec<cl_deferred_option> *) ia64_deferred_options;
6073
6074   if (v)
6075     FOR_EACH_VEC_ELT (*v, i, opt)
6076       {
6077         switch (opt->opt_index)
6078           {
6079           case OPT_mfixed_range_:
6080             fix_range (opt->arg);
6081             break;
6082
6083           default:
6084             gcc_unreachable ();
6085           }
6086       }
6087
6088   if (TARGET_AUTO_PIC)
6089     target_flags |= MASK_CONST_GP;
6090
6091   /* Numerous experiment shows that IRA based loop pressure
6092      calculation works better for RTL loop invariant motion on targets
6093      with enough (>= 32) registers.  It is an expensive optimization.
6094      So it is on only for peak performance.  */
6095   if (optimize >= 3)
6096     flag_ira_loop_pressure = 1;
6097
6098
6099   ia64_section_threshold = (global_options_set.x_g_switch_value
6100                             ? g_switch_value
6101                             : IA64_DEFAULT_GVALUE);
6102
6103   init_machine_status = ia64_init_machine_status;
6104
6105   if (flag_align_functions && !str_align_functions)
6106     str_align_functions = "64";
6107   if (flag_align_loops && !str_align_loops)
6108     str_align_loops = "32";
6109   if (TARGET_ABI_OPEN_VMS)
6110     flag_no_common = 1;
6111
6112   ia64_override_options_after_change();
6113 }
6114
6115 /* Implement targetm.override_options_after_change.  */
6116
6117 static void
6118 ia64_override_options_after_change (void)
6119 {
6120   if (optimize >= 3
6121       && !global_options_set.x_flag_selective_scheduling
6122       && !global_options_set.x_flag_selective_scheduling2)
6123     {
6124       flag_selective_scheduling2 = 1;
6125       flag_sel_sched_pipelining = 1;
6126     }
6127   if (mflag_sched_control_spec == 2)
6128     {
6129       /* Control speculation is on by default for the selective scheduler,
6130          but not for the Haifa scheduler.  */
6131       mflag_sched_control_spec = flag_selective_scheduling2 ? 1 : 0;
6132     }
6133   if (flag_sel_sched_pipelining && flag_auto_inc_dec)
6134     {
6135       /* FIXME: remove this when we'd implement breaking autoinsns as
6136          a transformation.  */
6137       flag_auto_inc_dec = 0;
6138     }
6139 }
6140
6141 /* Initialize the record of emitted frame related registers.  */
6142
6143 void ia64_init_expanders (void)
6144 {
6145   memset (&emitted_frame_related_regs, 0, sizeof (emitted_frame_related_regs));
6146 }
6147
6148 static struct machine_function *
6149 ia64_init_machine_status (void)
6150 {
6151   return ggc_cleared_alloc<machine_function> ();
6152 }
6153 \f
6154 static enum attr_itanium_class ia64_safe_itanium_class (rtx_insn *);
6155 static enum attr_type ia64_safe_type (rtx_insn *);
6156
6157 static enum attr_itanium_class
6158 ia64_safe_itanium_class (rtx_insn *insn)
6159 {
6160   if (recog_memoized (insn) >= 0)
6161     return get_attr_itanium_class (insn);
6162   else if (DEBUG_INSN_P (insn))
6163     return ITANIUM_CLASS_IGNORE;
6164   else
6165     return ITANIUM_CLASS_UNKNOWN;
6166 }
6167
6168 static enum attr_type
6169 ia64_safe_type (rtx_insn *insn)
6170 {
6171   if (recog_memoized (insn) >= 0)
6172     return get_attr_type (insn);
6173   else
6174     return TYPE_UNKNOWN;
6175 }
6176 \f
6177 /* The following collection of routines emit instruction group stop bits as
6178    necessary to avoid dependencies.  */
6179
6180 /* Need to track some additional registers as far as serialization is
6181    concerned so we can properly handle br.call and br.ret.  We could
6182    make these registers visible to gcc, but since these registers are
6183    never explicitly used in gcc generated code, it seems wasteful to
6184    do so (plus it would make the call and return patterns needlessly
6185    complex).  */
6186 #define REG_RP          (BR_REG (0))
6187 #define REG_AR_CFM      (FIRST_PSEUDO_REGISTER + 1)
6188 /* This is used for volatile asms which may require a stop bit immediately
6189    before and after them.  */
6190 #define REG_VOLATILE    (FIRST_PSEUDO_REGISTER + 2)
6191 #define AR_UNAT_BIT_0   (FIRST_PSEUDO_REGISTER + 3)
6192 #define NUM_REGS        (AR_UNAT_BIT_0 + 64)
6193
6194 /* For each register, we keep track of how it has been written in the
6195    current instruction group.
6196
6197    If a register is written unconditionally (no qualifying predicate),
6198    WRITE_COUNT is set to 2 and FIRST_PRED is ignored.
6199
6200    If a register is written if its qualifying predicate P is true, we
6201    set WRITE_COUNT to 1 and FIRST_PRED to P.  Later on, the same register
6202    may be written again by the complement of P (P^1) and when this happens,
6203    WRITE_COUNT gets set to 2.
6204
6205    The result of this is that whenever an insn attempts to write a register
6206    whose WRITE_COUNT is two, we need to issue an insn group barrier first.
6207
6208    If a predicate register is written by a floating-point insn, we set
6209    WRITTEN_BY_FP to true.
6210
6211    If a predicate register is written by an AND.ORCM we set WRITTEN_BY_AND
6212    to true; if it was written by an OR.ANDCM we set WRITTEN_BY_OR to true.  */
6213
6214 #if GCC_VERSION >= 4000
6215 #define RWS_FIELD_TYPE __extension__ unsigned short
6216 #else
6217 #define RWS_FIELD_TYPE unsigned int
6218 #endif
6219 struct reg_write_state
6220 {
6221   RWS_FIELD_TYPE write_count : 2;
6222   RWS_FIELD_TYPE first_pred : 10;
6223   RWS_FIELD_TYPE written_by_fp : 1;
6224   RWS_FIELD_TYPE written_by_and : 1;
6225   RWS_FIELD_TYPE written_by_or : 1;
6226 };
6227
6228 /* Cumulative info for the current instruction group.  */
6229 struct reg_write_state rws_sum[NUM_REGS];
6230 #if CHECKING_P
6231 /* Bitmap whether a register has been written in the current insn.  */
6232 HARD_REG_ELT_TYPE rws_insn[(NUM_REGS + HOST_BITS_PER_WIDEST_FAST_INT - 1)
6233                            / HOST_BITS_PER_WIDEST_FAST_INT];
6234
6235 static inline void
6236 rws_insn_set (int regno)
6237 {
6238   gcc_assert (!TEST_HARD_REG_BIT (rws_insn, regno));
6239   SET_HARD_REG_BIT (rws_insn, regno);
6240 }
6241
6242 static inline int
6243 rws_insn_test (int regno)
6244 {
6245   return TEST_HARD_REG_BIT (rws_insn, regno);
6246 }
6247 #else
6248 /* When not checking, track just REG_AR_CFM and REG_VOLATILE.  */
6249 unsigned char rws_insn[2];
6250
6251 static inline void
6252 rws_insn_set (int regno)
6253 {
6254   if (regno == REG_AR_CFM)
6255     rws_insn[0] = 1;
6256   else if (regno == REG_VOLATILE)
6257     rws_insn[1] = 1;
6258 }
6259
6260 static inline int
6261 rws_insn_test (int regno)
6262 {
6263   if (regno == REG_AR_CFM)
6264     return rws_insn[0];
6265   if (regno == REG_VOLATILE)
6266     return rws_insn[1];
6267   return 0;
6268 }
6269 #endif
6270
6271 /* Indicates whether this is the first instruction after a stop bit,
6272    in which case we don't need another stop bit.  Without this,
6273    ia64_variable_issue will die when scheduling an alloc.  */
6274 static int first_instruction;
6275
6276 /* Misc flags needed to compute RAW/WAW dependencies while we are traversing
6277    RTL for one instruction.  */
6278 struct reg_flags
6279 {
6280   unsigned int is_write : 1;    /* Is register being written?  */
6281   unsigned int is_fp : 1;       /* Is register used as part of an fp op?  */
6282   unsigned int is_branch : 1;   /* Is register used as part of a branch?  */
6283   unsigned int is_and : 1;      /* Is register used as part of and.orcm?  */
6284   unsigned int is_or : 1;       /* Is register used as part of or.andcm?  */
6285   unsigned int is_sibcall : 1;  /* Is this a sibling or normal call?  */
6286 };
6287
6288 static void rws_update (int, struct reg_flags, int);
6289 static int rws_access_regno (int, struct reg_flags, int);
6290 static int rws_access_reg (rtx, struct reg_flags, int);
6291 static void update_set_flags (rtx, struct reg_flags *);
6292 static int set_src_needs_barrier (rtx, struct reg_flags, int);
6293 static int rtx_needs_barrier (rtx, struct reg_flags, int);
6294 static void init_insn_group_barriers (void);
6295 static int group_barrier_needed (rtx_insn *);
6296 static int safe_group_barrier_needed (rtx_insn *);
6297 static int in_safe_group_barrier;
6298
6299 /* Update *RWS for REGNO, which is being written by the current instruction,
6300    with predicate PRED, and associated register flags in FLAGS.  */
6301
6302 static void
6303 rws_update (int regno, struct reg_flags flags, int pred)
6304 {
6305   if (pred)
6306     rws_sum[regno].write_count++;
6307   else
6308     rws_sum[regno].write_count = 2;
6309   rws_sum[regno].written_by_fp |= flags.is_fp;
6310   /* ??? Not tracking and/or across differing predicates.  */
6311   rws_sum[regno].written_by_and = flags.is_and;
6312   rws_sum[regno].written_by_or = flags.is_or;
6313   rws_sum[regno].first_pred = pred;
6314 }
6315
6316 /* Handle an access to register REGNO of type FLAGS using predicate register
6317    PRED.  Update rws_sum array.  Return 1 if this access creates
6318    a dependency with an earlier instruction in the same group.  */
6319
6320 static int
6321 rws_access_regno (int regno, struct reg_flags flags, int pred)
6322 {
6323   int need_barrier = 0;
6324
6325   gcc_assert (regno < NUM_REGS);
6326
6327   if (! PR_REGNO_P (regno))
6328     flags.is_and = flags.is_or = 0;
6329
6330   if (flags.is_write)
6331     {
6332       int write_count;
6333
6334       rws_insn_set (regno);
6335       write_count = rws_sum[regno].write_count;
6336
6337       switch (write_count)
6338         {
6339         case 0:
6340           /* The register has not been written yet.  */
6341           if (!in_safe_group_barrier)
6342             rws_update (regno, flags, pred);
6343           break;
6344
6345         case 1:
6346           /* The register has been written via a predicate.  Treat
6347              it like a unconditional write and do not try to check
6348              for complementary pred reg in earlier write.  */
6349           if (flags.is_and && rws_sum[regno].written_by_and)
6350             ;
6351           else if (flags.is_or && rws_sum[regno].written_by_or)
6352             ;
6353           else
6354             need_barrier = 1;
6355           if (!in_safe_group_barrier)
6356             rws_update (regno, flags, pred);
6357           break;
6358
6359         case 2:
6360           /* The register has been unconditionally written already.  We
6361              need a barrier.  */
6362           if (flags.is_and && rws_sum[regno].written_by_and)
6363             ;
6364           else if (flags.is_or && rws_sum[regno].written_by_or)
6365             ;
6366           else
6367             need_barrier = 1;
6368           if (!in_safe_group_barrier)
6369             {
6370               rws_sum[regno].written_by_and = flags.is_and;
6371               rws_sum[regno].written_by_or = flags.is_or;
6372             }
6373           break;
6374
6375         default:
6376           gcc_unreachable ();
6377         }
6378     }
6379   else
6380     {
6381       if (flags.is_branch)
6382         {
6383           /* Branches have several RAW exceptions that allow to avoid
6384              barriers.  */
6385
6386           if (REGNO_REG_CLASS (regno) == BR_REGS || regno == AR_PFS_REGNUM)
6387             /* RAW dependencies on branch regs are permissible as long
6388                as the writer is a non-branch instruction.  Since we
6389                never generate code that uses a branch register written
6390                by a branch instruction, handling this case is
6391                easy.  */
6392             return 0;
6393
6394           if (REGNO_REG_CLASS (regno) == PR_REGS
6395               && ! rws_sum[regno].written_by_fp)
6396             /* The predicates of a branch are available within the
6397                same insn group as long as the predicate was written by
6398                something other than a floating-point instruction.  */
6399             return 0;
6400         }
6401
6402       if (flags.is_and && rws_sum[regno].written_by_and)
6403         return 0;
6404       if (flags.is_or && rws_sum[regno].written_by_or)
6405         return 0;
6406
6407       switch (rws_sum[regno].write_count)
6408         {
6409         case 0:
6410           /* The register has not been written yet.  */
6411           break;
6412
6413         case 1:
6414           /* The register has been written via a predicate, assume we
6415              need a barrier (don't check for complementary regs).  */
6416           need_barrier = 1;
6417           break;
6418
6419         case 2:
6420           /* The register has been unconditionally written already.  We
6421              need a barrier.  */
6422           need_barrier = 1;
6423           break;
6424
6425         default:
6426           gcc_unreachable ();
6427         }
6428     }
6429
6430   return need_barrier;
6431 }
6432
6433 static int
6434 rws_access_reg (rtx reg, struct reg_flags flags, int pred)
6435 {
6436   int regno = REGNO (reg);
6437   int n = REG_NREGS (reg);
6438
6439   if (n == 1)
6440     return rws_access_regno (regno, flags, pred);
6441   else
6442     {
6443       int need_barrier = 0;
6444       while (--n >= 0)
6445         need_barrier |= rws_access_regno (regno + n, flags, pred);
6446       return need_barrier;
6447     }
6448 }
6449
6450 /* Examine X, which is a SET rtx, and update the flags, the predicate, and
6451    the condition, stored in *PFLAGS, *PPRED and *PCOND.  */
6452
6453 static void
6454 update_set_flags (rtx x, struct reg_flags *pflags)
6455 {
6456   rtx src = SET_SRC (x);
6457
6458   switch (GET_CODE (src))
6459     {
6460     case CALL:
6461       return;
6462
6463     case IF_THEN_ELSE:
6464       /* There are four cases here:
6465          (1) The destination is (pc), in which case this is a branch,
6466          nothing here applies.
6467          (2) The destination is ar.lc, in which case this is a
6468          doloop_end_internal,
6469          (3) The destination is an fp register, in which case this is
6470          an fselect instruction.
6471          (4) The condition has (unspec [(reg)] UNSPEC_LDC), in which case
6472          this is a check load.
6473          In all cases, nothing we do in this function applies.  */
6474       return;
6475
6476     default:
6477       if (COMPARISON_P (src)
6478           && SCALAR_FLOAT_MODE_P (GET_MODE (XEXP (src, 0))))
6479         /* Set pflags->is_fp to 1 so that we know we're dealing
6480            with a floating point comparison when processing the
6481            destination of the SET.  */
6482         pflags->is_fp = 1;
6483
6484       /* Discover if this is a parallel comparison.  We only handle
6485          and.orcm and or.andcm at present, since we must retain a
6486          strict inverse on the predicate pair.  */
6487       else if (GET_CODE (src) == AND)
6488         pflags->is_and = 1;
6489       else if (GET_CODE (src) == IOR)
6490         pflags->is_or = 1;
6491
6492       break;
6493     }
6494 }
6495
6496 /* Subroutine of rtx_needs_barrier; this function determines whether the
6497    source of a given SET rtx found in X needs a barrier.  FLAGS and PRED
6498    are as in rtx_needs_barrier.  COND is an rtx that holds the condition
6499    for this insn.  */
6500
6501 static int
6502 set_src_needs_barrier (rtx x, struct reg_flags flags, int pred)
6503 {
6504   int need_barrier = 0;
6505   rtx dst;
6506   rtx src = SET_SRC (x);
6507
6508   if (GET_CODE (src) == CALL)
6509     /* We don't need to worry about the result registers that
6510        get written by subroutine call.  */
6511     return rtx_needs_barrier (src, flags, pred);
6512   else if (SET_DEST (x) == pc_rtx)
6513     {
6514       /* X is a conditional branch.  */
6515       /* ??? This seems redundant, as the caller sets this bit for
6516          all JUMP_INSNs.  */
6517       if (!ia64_spec_check_src_p (src))
6518         flags.is_branch = 1;
6519       return rtx_needs_barrier (src, flags, pred);
6520     }
6521
6522   if (ia64_spec_check_src_p (src))
6523     /* Avoid checking one register twice (in condition
6524        and in 'then' section) for ldc pattern.  */
6525     {
6526       gcc_assert (REG_P (XEXP (src, 2)));
6527       need_barrier = rtx_needs_barrier (XEXP (src, 2), flags, pred);
6528
6529       /* We process MEM below.  */
6530       src = XEXP (src, 1);
6531     }
6532
6533   need_barrier |= rtx_needs_barrier (src, flags, pred);
6534
6535   dst = SET_DEST (x);
6536   if (GET_CODE (dst) == ZERO_EXTRACT)
6537     {
6538       need_barrier |= rtx_needs_barrier (XEXP (dst, 1), flags, pred);
6539       need_barrier |= rtx_needs_barrier (XEXP (dst, 2), flags, pred);
6540     }
6541   return need_barrier;
6542 }
6543
6544 /* Handle an access to rtx X of type FLAGS using predicate register
6545    PRED.  Return 1 if this access creates a dependency with an earlier
6546    instruction in the same group.  */
6547
6548 static int
6549 rtx_needs_barrier (rtx x, struct reg_flags flags, int pred)
6550 {
6551   int i, j;
6552   int is_complemented = 0;
6553   int need_barrier = 0;
6554   const char *format_ptr;
6555   struct reg_flags new_flags;
6556   rtx cond;
6557
6558   if (! x)
6559     return 0;
6560
6561   new_flags = flags;
6562
6563   switch (GET_CODE (x))
6564     {
6565     case SET:
6566       update_set_flags (x, &new_flags);
6567       need_barrier = set_src_needs_barrier (x, new_flags, pred);
6568       if (GET_CODE (SET_SRC (x)) != CALL)
6569         {
6570           new_flags.is_write = 1;
6571           need_barrier |= rtx_needs_barrier (SET_DEST (x), new_flags, pred);
6572         }
6573       break;
6574
6575     case CALL:
6576       new_flags.is_write = 0;
6577       need_barrier |= rws_access_regno (AR_EC_REGNUM, new_flags, pred);
6578
6579       /* Avoid multiple register writes, in case this is a pattern with
6580          multiple CALL rtx.  This avoids a failure in rws_access_reg.  */
6581       if (! flags.is_sibcall && ! rws_insn_test (REG_AR_CFM))
6582         {
6583           new_flags.is_write = 1;
6584           need_barrier |= rws_access_regno (REG_RP, new_flags, pred);
6585           need_barrier |= rws_access_regno (AR_PFS_REGNUM, new_flags, pred);
6586           need_barrier |= rws_access_regno (REG_AR_CFM, new_flags, pred);
6587         }
6588       break;
6589
6590     case COND_EXEC:
6591       /* X is a predicated instruction.  */
6592
6593       cond = COND_EXEC_TEST (x);
6594       gcc_assert (!pred);
6595       need_barrier = rtx_needs_barrier (cond, flags, 0);
6596
6597       if (GET_CODE (cond) == EQ)
6598         is_complemented = 1;
6599       cond = XEXP (cond, 0);
6600       gcc_assert (GET_CODE (cond) == REG
6601                   && REGNO_REG_CLASS (REGNO (cond)) == PR_REGS);
6602       pred = REGNO (cond);
6603       if (is_complemented)
6604         ++pred;
6605
6606       need_barrier |= rtx_needs_barrier (COND_EXEC_CODE (x), flags, pred);
6607       return need_barrier;
6608
6609     case CLOBBER:
6610     case USE:
6611       /* Clobber & use are for earlier compiler-phases only.  */
6612       break;
6613
6614     case ASM_OPERANDS:
6615     case ASM_INPUT:
6616       /* We always emit stop bits for traditional asms.  We emit stop bits
6617          for volatile extended asms if TARGET_VOL_ASM_STOP is true.  */
6618       if (GET_CODE (x) != ASM_OPERANDS
6619           || (MEM_VOLATILE_P (x) && TARGET_VOL_ASM_STOP))
6620         {
6621           /* Avoid writing the register multiple times if we have multiple
6622              asm outputs.  This avoids a failure in rws_access_reg.  */
6623           if (! rws_insn_test (REG_VOLATILE))
6624             {
6625               new_flags.is_write = 1;
6626               rws_access_regno (REG_VOLATILE, new_flags, pred);
6627             }
6628           return 1;
6629         }
6630
6631       /* For all ASM_OPERANDS, we must traverse the vector of input operands.
6632          We cannot just fall through here since then we would be confused
6633          by the ASM_INPUT rtx inside ASM_OPERANDS, which do not indicate
6634          traditional asms unlike their normal usage.  */
6635
6636       for (i = ASM_OPERANDS_INPUT_LENGTH (x) - 1; i >= 0; --i)
6637         if (rtx_needs_barrier (ASM_OPERANDS_INPUT (x, i), flags, pred))
6638           need_barrier = 1;
6639       break;
6640
6641     case PARALLEL:
6642       for (i = XVECLEN (x, 0) - 1; i >= 0; --i)
6643         {
6644           rtx pat = XVECEXP (x, 0, i);
6645           switch (GET_CODE (pat))
6646             {
6647             case SET:
6648               update_set_flags (pat, &new_flags);
6649               need_barrier |= set_src_needs_barrier (pat, new_flags, pred);
6650               break;
6651
6652             case USE:
6653             case CALL:
6654             case ASM_OPERANDS:
6655             case ASM_INPUT:
6656               need_barrier |= rtx_needs_barrier (pat, flags, pred);
6657               break;
6658
6659             case CLOBBER:
6660               if (REG_P (XEXP (pat, 0))
6661                   && extract_asm_operands (x) != NULL_RTX
6662                   && REGNO (XEXP (pat, 0)) != AR_UNAT_REGNUM)
6663                 {
6664                   new_flags.is_write = 1;
6665                   need_barrier |= rtx_needs_barrier (XEXP (pat, 0),
6666                                                      new_flags, pred);
6667                   new_flags = flags;
6668                 }
6669               break;
6670
6671             case RETURN:
6672               break;
6673
6674             default:
6675               gcc_unreachable ();
6676             }
6677         }
6678       for (i = XVECLEN (x, 0) - 1; i >= 0; --i)
6679         {
6680           rtx pat = XVECEXP (x, 0, i);
6681           if (GET_CODE (pat) == SET)
6682             {
6683               if (GET_CODE (SET_SRC (pat)) != CALL)
6684                 {
6685                   new_flags.is_write = 1;
6686                   need_barrier |= rtx_needs_barrier (SET_DEST (pat), new_flags,
6687                                                      pred);
6688                 }
6689             }
6690           else if (GET_CODE (pat) == CLOBBER || GET_CODE (pat) == RETURN)
6691             need_barrier |= rtx_needs_barrier (pat, flags, pred);
6692         }
6693       break;
6694
6695     case SUBREG:
6696       need_barrier |= rtx_needs_barrier (SUBREG_REG (x), flags, pred);
6697       break;
6698     case REG:
6699       if (REGNO (x) == AR_UNAT_REGNUM)
6700         {
6701           for (i = 0; i < 64; ++i)
6702             need_barrier |= rws_access_regno (AR_UNAT_BIT_0 + i, flags, pred);
6703         }
6704       else
6705         need_barrier = rws_access_reg (x, flags, pred);
6706       break;
6707
6708     case MEM:
6709       /* Find the regs used in memory address computation.  */
6710       new_flags.is_write = 0;
6711       need_barrier = rtx_needs_barrier (XEXP (x, 0), new_flags, pred);
6712       break;
6713
6714     case CONST_INT:   case CONST_DOUBLE:  case CONST_VECTOR:
6715     case SYMBOL_REF:  case LABEL_REF:     case CONST:
6716       break;
6717
6718       /* Operators with side-effects.  */
6719     case POST_INC:    case POST_DEC:
6720       gcc_assert (GET_CODE (XEXP (x, 0)) == REG);
6721
6722       new_flags.is_write = 0;
6723       need_barrier  = rws_access_reg (XEXP (x, 0), new_flags, pred);
6724       new_flags.is_write = 1;
6725       need_barrier |= rws_access_reg (XEXP (x, 0), new_flags, pred);
6726       break;
6727
6728     case POST_MODIFY:
6729       gcc_assert (GET_CODE (XEXP (x, 0)) == REG);
6730
6731       new_flags.is_write = 0;
6732       need_barrier  = rws_access_reg (XEXP (x, 0), new_flags, pred);
6733       need_barrier |= rtx_needs_barrier (XEXP (x, 1), new_flags, pred);
6734       new_flags.is_write = 1;
6735       need_barrier |= rws_access_reg (XEXP (x, 0), new_flags, pred);
6736       break;
6737
6738       /* Handle common unary and binary ops for efficiency.  */
6739     case COMPARE:  case PLUS:    case MINUS:   case MULT:      case DIV:
6740     case MOD:      case UDIV:    case UMOD:    case AND:       case IOR:
6741     case XOR:      case ASHIFT:  case ROTATE:  case ASHIFTRT:  case LSHIFTRT:
6742     case ROTATERT: case SMIN:    case SMAX:    case UMIN:      case UMAX:
6743     case NE:       case EQ:      case GE:      case GT:        case LE:
6744     case LT:       case GEU:     case GTU:     case LEU:       case LTU:
6745       need_barrier = rtx_needs_barrier (XEXP (x, 0), new_flags, pred);
6746       need_barrier |= rtx_needs_barrier (XEXP (x, 1), new_flags, pred);
6747       break;
6748
6749     case NEG:      case NOT:            case SIGN_EXTEND:     case ZERO_EXTEND:
6750     case TRUNCATE: case FLOAT_EXTEND:   case FLOAT_TRUNCATE:  case FLOAT:
6751     case FIX:      case UNSIGNED_FLOAT: case UNSIGNED_FIX:    case ABS:
6752     case SQRT:     case FFS:            case POPCOUNT:
6753       need_barrier = rtx_needs_barrier (XEXP (x, 0), flags, pred);
6754       break;
6755
6756     case VEC_SELECT:
6757       /* VEC_SELECT's second argument is a PARALLEL with integers that
6758          describe the elements selected.  On ia64, those integers are
6759          always constants.  Avoid walking the PARALLEL so that we don't
6760          get confused with "normal" parallels and then die.  */
6761       need_barrier = rtx_needs_barrier (XEXP (x, 0), flags, pred);
6762       break;
6763
6764     case UNSPEC:
6765       switch (XINT (x, 1))
6766         {
6767         case UNSPEC_LTOFF_DTPMOD:
6768         case UNSPEC_LTOFF_DTPREL:
6769         case UNSPEC_DTPREL:
6770         case UNSPEC_LTOFF_TPREL:
6771         case UNSPEC_TPREL:
6772         case UNSPEC_PRED_REL_MUTEX:
6773         case UNSPEC_PIC_CALL:
6774         case UNSPEC_MF:
6775         case UNSPEC_FETCHADD_ACQ:
6776         case UNSPEC_FETCHADD_REL:
6777         case UNSPEC_BSP_VALUE:
6778         case UNSPEC_FLUSHRS:
6779         case UNSPEC_BUNDLE_SELECTOR:
6780           break;
6781
6782         case UNSPEC_GR_SPILL:
6783         case UNSPEC_GR_RESTORE:
6784           {
6785             HOST_WIDE_INT offset = INTVAL (XVECEXP (x, 0, 1));
6786             HOST_WIDE_INT bit = (offset >> 3) & 63;
6787
6788             need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6789             new_flags.is_write = (XINT (x, 1) == UNSPEC_GR_SPILL);
6790             need_barrier |= rws_access_regno (AR_UNAT_BIT_0 + bit,
6791                                               new_flags, pred);
6792             break;
6793           }
6794
6795         case UNSPEC_FR_SPILL:
6796         case UNSPEC_FR_RESTORE:
6797         case UNSPEC_GETF_EXP:
6798         case UNSPEC_SETF_EXP:
6799         case UNSPEC_ADDP4:
6800         case UNSPEC_FR_SQRT_RECIP_APPROX:
6801         case UNSPEC_FR_SQRT_RECIP_APPROX_RES:
6802         case UNSPEC_LDA:
6803         case UNSPEC_LDS:
6804         case UNSPEC_LDS_A:
6805         case UNSPEC_LDSA:
6806         case UNSPEC_CHKACLR:
6807         case UNSPEC_CHKS:
6808           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6809           break;
6810
6811         case UNSPEC_FR_RECIP_APPROX:
6812         case UNSPEC_SHRP:
6813         case UNSPEC_COPYSIGN:
6814         case UNSPEC_FR_RECIP_APPROX_RES:
6815           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6816           need_barrier |= rtx_needs_barrier (XVECEXP (x, 0, 1), flags, pred);
6817           break;
6818
6819         case UNSPEC_CMPXCHG_ACQ:
6820         case UNSPEC_CMPXCHG_REL:
6821           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 1), flags, pred);
6822           need_barrier |= rtx_needs_barrier (XVECEXP (x, 0, 2), flags, pred);
6823           break;
6824
6825         default:
6826           gcc_unreachable ();
6827         }
6828       break;
6829
6830     case UNSPEC_VOLATILE:
6831       switch (XINT (x, 1))
6832         {
6833         case UNSPECV_ALLOC:
6834           /* Alloc must always be the first instruction of a group.
6835              We force this by always returning true.  */
6836           /* ??? We might get better scheduling if we explicitly check for
6837              input/local/output register dependencies, and modify the
6838              scheduler so that alloc is always reordered to the start of
6839              the current group.  We could then eliminate all of the
6840              first_instruction code.  */
6841           rws_access_regno (AR_PFS_REGNUM, flags, pred);
6842
6843           new_flags.is_write = 1;
6844           rws_access_regno (REG_AR_CFM, new_flags, pred);
6845           return 1;
6846
6847         case UNSPECV_SET_BSP:
6848         case UNSPECV_PROBE_STACK_RANGE:
6849           need_barrier = 1;
6850           break;
6851
6852         case UNSPECV_BLOCKAGE:
6853         case UNSPECV_INSN_GROUP_BARRIER:
6854         case UNSPECV_BREAK:
6855         case UNSPECV_PSAC_ALL:
6856         case UNSPECV_PSAC_NORMAL:
6857           return 0;
6858
6859         case UNSPECV_PROBE_STACK_ADDRESS:
6860           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6861           break;
6862
6863         default:
6864           gcc_unreachable ();
6865         }
6866       break;
6867
6868     case RETURN:
6869       new_flags.is_write = 0;
6870       need_barrier  = rws_access_regno (REG_RP, flags, pred);
6871       need_barrier |= rws_access_regno (AR_PFS_REGNUM, flags, pred);
6872
6873       new_flags.is_write = 1;
6874       need_barrier |= rws_access_regno (AR_EC_REGNUM, new_flags, pred);
6875       need_barrier |= rws_access_regno (REG_AR_CFM, new_flags, pred);
6876       break;
6877
6878     default:
6879       format_ptr = GET_RTX_FORMAT (GET_CODE (x));
6880       for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
6881         switch (format_ptr[i])
6882           {
6883           case '0':     /* unused field */
6884           case 'i':     /* integer */
6885           case 'n':     /* note */
6886           case 'w':     /* wide integer */
6887           case 's':     /* pointer to string */
6888           case 'S':     /* optional pointer to string */
6889             break;
6890
6891           case 'e':
6892             if (rtx_needs_barrier (XEXP (x, i), flags, pred))
6893               need_barrier = 1;
6894             break;
6895
6896           case 'E':
6897             for (j = XVECLEN (x, i) - 1; j >= 0; --j)
6898               if (rtx_needs_barrier (XVECEXP (x, i, j), flags, pred))
6899                 need_barrier = 1;
6900             break;
6901
6902           default:
6903             gcc_unreachable ();
6904           }
6905       break;
6906     }
6907   return need_barrier;
6908 }
6909
6910 /* Clear out the state for group_barrier_needed at the start of a
6911    sequence of insns.  */
6912
6913 static void
6914 init_insn_group_barriers (void)
6915 {
6916   memset (rws_sum, 0, sizeof (rws_sum));
6917   first_instruction = 1;
6918 }
6919
6920 /* Given the current state, determine whether a group barrier (a stop bit) is
6921    necessary before INSN.  Return nonzero if so.  This modifies the state to
6922    include the effects of INSN as a side-effect.  */
6923
6924 static int
6925 group_barrier_needed (rtx_insn *insn)
6926 {
6927   rtx pat;
6928   int need_barrier = 0;
6929   struct reg_flags flags;
6930
6931   memset (&flags, 0, sizeof (flags));
6932   switch (GET_CODE (insn))
6933     {
6934     case NOTE:
6935     case DEBUG_INSN:
6936       break;
6937
6938     case BARRIER:
6939       /* A barrier doesn't imply an instruction group boundary.  */
6940       break;
6941
6942     case CODE_LABEL:
6943       memset (rws_insn, 0, sizeof (rws_insn));
6944       return 1;
6945
6946     case CALL_INSN:
6947       flags.is_branch = 1;
6948       flags.is_sibcall = SIBLING_CALL_P (insn);
6949       memset (rws_insn, 0, sizeof (rws_insn));
6950
6951       /* Don't bundle a call following another call.  */
6952       if ((pat = prev_active_insn (insn)) && CALL_P (pat))
6953         {
6954           need_barrier = 1;
6955           break;
6956         }
6957
6958       need_barrier = rtx_needs_barrier (PATTERN (insn), flags, 0);
6959       break;
6960
6961     case JUMP_INSN:
6962       if (!ia64_spec_check_p (insn))
6963         flags.is_branch = 1;
6964
6965       /* Don't bundle a jump following a call.  */
6966       if ((pat = prev_active_insn (insn)) && CALL_P (pat))
6967         {
6968           need_barrier = 1;
6969           break;
6970         }
6971       /* FALLTHRU */
6972
6973     case INSN:
6974       if (GET_CODE (PATTERN (insn)) == USE
6975           || GET_CODE (PATTERN (insn)) == CLOBBER)
6976         /* Don't care about USE and CLOBBER "insns"---those are used to
6977            indicate to the optimizer that it shouldn't get rid of
6978            certain operations.  */
6979         break;
6980
6981       pat = PATTERN (insn);
6982
6983       /* Ug.  Hack hacks hacked elsewhere.  */
6984       switch (recog_memoized (insn))
6985         {
6986           /* We play dependency tricks with the epilogue in order
6987              to get proper schedules.  Undo this for dv analysis.  */
6988         case CODE_FOR_epilogue_deallocate_stack:
6989         case CODE_FOR_prologue_allocate_stack:
6990           pat = XVECEXP (pat, 0, 0);
6991           break;
6992
6993           /* The pattern we use for br.cloop confuses the code above.
6994              The second element of the vector is representative.  */
6995         case CODE_FOR_doloop_end_internal:
6996           pat = XVECEXP (pat, 0, 1);
6997           break;
6998
6999           /* Doesn't generate code.  */
7000         case CODE_FOR_pred_rel_mutex:
7001         case CODE_FOR_prologue_use:
7002           return 0;
7003
7004         default:
7005           break;
7006         }
7007
7008       memset (rws_insn, 0, sizeof (rws_insn));
7009       need_barrier = rtx_needs_barrier (pat, flags, 0);
7010
7011       /* Check to see if the previous instruction was a volatile
7012          asm.  */
7013       if (! need_barrier)
7014         need_barrier = rws_access_regno (REG_VOLATILE, flags, 0);
7015
7016       break;
7017
7018     default:
7019       gcc_unreachable ();
7020     }
7021
7022   if (first_instruction && important_for_bundling_p (insn))
7023     {
7024       need_barrier = 0;
7025       first_instruction = 0;
7026     }
7027
7028   return need_barrier;
7029 }
7030
7031 /* Like group_barrier_needed, but do not clobber the current state.  */
7032
7033 static int
7034 safe_group_barrier_needed (rtx_insn *insn)
7035 {
7036   int saved_first_instruction;
7037   int t;
7038
7039   saved_first_instruction = first_instruction;
7040   in_safe_group_barrier = 1;
7041
7042   t = group_barrier_needed (insn);
7043
7044   first_instruction = saved_first_instruction;
7045   in_safe_group_barrier = 0;
7046
7047   return t;
7048 }
7049
7050 /* Scan the current function and insert stop bits as necessary to
7051    eliminate dependencies.  This function assumes that a final
7052    instruction scheduling pass has been run which has already
7053    inserted most of the necessary stop bits.  This function only
7054    inserts new ones at basic block boundaries, since these are
7055    invisible to the scheduler.  */
7056
7057 static void
7058 emit_insn_group_barriers (FILE *dump)
7059 {
7060   rtx_insn *insn;
7061   rtx_insn *last_label = 0;
7062   int insns_since_last_label = 0;
7063
7064   init_insn_group_barriers ();
7065
7066   for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
7067     {
7068       if (LABEL_P (insn))
7069         {
7070           if (insns_since_last_label)
7071             last_label = insn;
7072           insns_since_last_label = 0;
7073         }
7074       else if (NOTE_P (insn)
7075                && NOTE_KIND (insn) == NOTE_INSN_BASIC_BLOCK)
7076         {
7077           if (insns_since_last_label)
7078             last_label = insn;
7079           insns_since_last_label = 0;
7080         }
7081       else if (NONJUMP_INSN_P (insn)
7082                && GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
7083                && XINT (PATTERN (insn), 1) == UNSPECV_INSN_GROUP_BARRIER)
7084         {
7085           init_insn_group_barriers ();
7086           last_label = 0;
7087         }
7088       else if (NONDEBUG_INSN_P (insn))
7089         {
7090           insns_since_last_label = 1;
7091
7092           if (group_barrier_needed (insn))
7093             {
7094               if (last_label)
7095                 {
7096                   if (dump)
7097                     fprintf (dump, "Emitting stop before label %d\n",
7098                              INSN_UID (last_label));
7099                   emit_insn_before (gen_insn_group_barrier (GEN_INT (3)), last_label);
7100                   insn = last_label;
7101
7102                   init_insn_group_barriers ();
7103                   last_label = 0;
7104                 }
7105             }
7106         }
7107     }
7108 }
7109
7110 /* Like emit_insn_group_barriers, but run if no final scheduling pass was run.
7111    This function has to emit all necessary group barriers.  */
7112
7113 static void
7114 emit_all_insn_group_barriers (FILE *dump ATTRIBUTE_UNUSED)
7115 {
7116   rtx_insn *insn;
7117
7118   init_insn_group_barriers ();
7119
7120   for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
7121     {
7122       if (BARRIER_P (insn))
7123         {
7124           rtx_insn *last = prev_active_insn (insn);
7125
7126           if (! last)
7127             continue;
7128           if (JUMP_TABLE_DATA_P (last))
7129             last = prev_active_insn (last);
7130           if (recog_memoized (last) != CODE_FOR_insn_group_barrier)
7131             emit_insn_after (gen_insn_group_barrier (GEN_INT (3)), last);
7132
7133           init_insn_group_barriers ();
7134         }
7135       else if (NONDEBUG_INSN_P (insn))
7136         {
7137           if (recog_memoized (insn) == CODE_FOR_insn_group_barrier)
7138             init_insn_group_barriers ();
7139           else if (group_barrier_needed (insn))
7140             {
7141               emit_insn_before (gen_insn_group_barrier (GEN_INT (3)), insn);
7142               init_insn_group_barriers ();
7143               group_barrier_needed (insn);
7144             }
7145         }
7146     }
7147 }
7148
7149 \f
7150
7151 /* Instruction scheduling support.  */
7152
7153 #define NR_BUNDLES 10
7154
7155 /* A list of names of all available bundles.  */
7156
7157 static const char *bundle_name [NR_BUNDLES] =
7158 {
7159   ".mii",
7160   ".mmi",
7161   ".mfi",
7162   ".mmf",
7163 #if NR_BUNDLES == 10
7164   ".bbb",
7165   ".mbb",
7166 #endif
7167   ".mib",
7168   ".mmb",
7169   ".mfb",
7170   ".mlx"
7171 };
7172
7173 /* Nonzero if we should insert stop bits into the schedule.  */
7174
7175 int ia64_final_schedule = 0;
7176
7177 /* Codes of the corresponding queried units: */
7178
7179 static int _0mii_, _0mmi_, _0mfi_, _0mmf_;
7180 static int _0bbb_, _0mbb_, _0mib_, _0mmb_, _0mfb_, _0mlx_;
7181
7182 static int _1mii_, _1mmi_, _1mfi_, _1mmf_;
7183 static int _1bbb_, _1mbb_, _1mib_, _1mmb_, _1mfb_, _1mlx_;
7184
7185 static int pos_1, pos_2, pos_3, pos_4, pos_5, pos_6;
7186
7187 /* The following variable value is an insn group barrier.  */
7188
7189 static rtx_insn *dfa_stop_insn;
7190
7191 /* The following variable value is the last issued insn.  */
7192
7193 static rtx_insn *last_scheduled_insn;
7194
7195 /* The following variable value is pointer to a DFA state used as
7196    temporary variable.  */
7197
7198 static state_t temp_dfa_state = NULL;
7199
7200 /* The following variable value is DFA state after issuing the last
7201    insn.  */
7202
7203 static state_t prev_cycle_state = NULL;
7204
7205 /* The following array element values are TRUE if the corresponding
7206    insn requires to add stop bits before it.  */
7207
7208 static char *stops_p = NULL;
7209
7210 /* The following variable is used to set up the mentioned above array.  */
7211
7212 static int stop_before_p = 0;
7213
7214 /* The following variable value is length of the arrays `clocks' and
7215    `add_cycles'. */
7216
7217 static int clocks_length;
7218
7219 /* The following variable value is number of data speculations in progress.  */
7220 static int pending_data_specs = 0;
7221
7222 /* Number of memory references on current and three future processor cycles.  */
7223 static char mem_ops_in_group[4];
7224
7225 /* Number of current processor cycle (from scheduler's point of view).  */
7226 static int current_cycle;
7227
7228 static rtx ia64_single_set (rtx_insn *);
7229 static void ia64_emit_insn_before (rtx, rtx_insn *);
7230
7231 /* Map a bundle number to its pseudo-op.  */
7232
7233 const char *
7234 get_bundle_name (int b)
7235 {
7236   return bundle_name[b];
7237 }
7238
7239
7240 /* Return the maximum number of instructions a cpu can issue.  */
7241
7242 static int
7243 ia64_issue_rate (void)
7244 {
7245   return 6;
7246 }
7247
7248 /* Helper function - like single_set, but look inside COND_EXEC.  */
7249
7250 static rtx
7251 ia64_single_set (rtx_insn *insn)
7252 {
7253   rtx x = PATTERN (insn), ret;
7254   if (GET_CODE (x) == COND_EXEC)
7255     x = COND_EXEC_CODE (x);
7256   if (GET_CODE (x) == SET)
7257     return x;
7258
7259   /* Special case here prologue_allocate_stack and epilogue_deallocate_stack.
7260      Although they are not classical single set, the second set is there just
7261      to protect it from moving past FP-relative stack accesses.  */
7262   switch (recog_memoized (insn))
7263     {
7264     case CODE_FOR_prologue_allocate_stack:
7265     case CODE_FOR_prologue_allocate_stack_pr:
7266     case CODE_FOR_epilogue_deallocate_stack:
7267     case CODE_FOR_epilogue_deallocate_stack_pr:
7268       ret = XVECEXP (x, 0, 0);
7269       break;
7270
7271     default:
7272       ret = single_set_2 (insn, x);
7273       break;
7274     }
7275
7276   return ret;
7277 }
7278
7279 /* Adjust the cost of a scheduling dependency.
7280    Return the new cost of a dependency of type DEP_TYPE or INSN on DEP_INSN.
7281    COST is the current cost, DW is dependency weakness.  */
7282 static int
7283 ia64_adjust_cost (rtx_insn *insn, int dep_type1, rtx_insn *dep_insn,
7284                   int cost, dw_t dw)
7285 {
7286   enum reg_note dep_type = (enum reg_note) dep_type1;
7287   enum attr_itanium_class dep_class;
7288   enum attr_itanium_class insn_class;
7289
7290   insn_class = ia64_safe_itanium_class (insn);
7291   dep_class = ia64_safe_itanium_class (dep_insn);
7292
7293   /* Treat true memory dependencies separately.  Ignore apparent true
7294      dependence between store and call (call has a MEM inside a SYMBOL_REF).  */
7295   if (dep_type == REG_DEP_TRUE
7296       && (dep_class == ITANIUM_CLASS_ST || dep_class == ITANIUM_CLASS_STF)
7297       && (insn_class == ITANIUM_CLASS_BR || insn_class == ITANIUM_CLASS_SCALL))
7298     return 0;
7299
7300   if (dw == MIN_DEP_WEAK)
7301     /* Store and load are likely to alias, use higher cost to avoid stall.  */
7302     return PARAM_VALUE (PARAM_SCHED_MEM_TRUE_DEP_COST);
7303   else if (dw > MIN_DEP_WEAK)
7304     {
7305       /* Store and load are less likely to alias.  */
7306       if (mflag_sched_fp_mem_deps_zero_cost && dep_class == ITANIUM_CLASS_STF)
7307         /* Assume there will be no cache conflict for floating-point data.
7308            For integer data, L1 conflict penalty is huge (17 cycles), so we
7309            never assume it will not cause a conflict.  */
7310         return 0;
7311       else
7312         return cost;
7313     }
7314
7315   if (dep_type != REG_DEP_OUTPUT)
7316     return cost;
7317
7318   if (dep_class == ITANIUM_CLASS_ST || dep_class == ITANIUM_CLASS_STF
7319       || insn_class == ITANIUM_CLASS_ST || insn_class == ITANIUM_CLASS_STF)
7320     return 0;
7321
7322   return cost;
7323 }
7324
7325 /* Like emit_insn_before, but skip cycle_display notes.
7326    ??? When cycle display notes are implemented, update this.  */
7327
7328 static void
7329 ia64_emit_insn_before (rtx insn, rtx_insn *before)
7330 {
7331   emit_insn_before (insn, before);
7332 }
7333
7334 /* The following function marks insns who produce addresses for load
7335    and store insns.  Such insns will be placed into M slots because it
7336    decrease latency time for Itanium1 (see function
7337    `ia64_produce_address_p' and the DFA descriptions).  */
7338
7339 static void
7340 ia64_dependencies_evaluation_hook (rtx_insn *head, rtx_insn *tail)
7341 {
7342   rtx_insn *insn, *next, *next_tail;
7343
7344   /* Before reload, which_alternative is not set, which means that
7345      ia64_safe_itanium_class will produce wrong results for (at least)
7346      move instructions.  */
7347   if (!reload_completed)
7348     return;
7349
7350   next_tail = NEXT_INSN (tail);
7351   for (insn = head; insn != next_tail; insn = NEXT_INSN (insn))
7352     if (INSN_P (insn))
7353       insn->call = 0;
7354   for (insn = head; insn != next_tail; insn = NEXT_INSN (insn))
7355     if (INSN_P (insn)
7356         && ia64_safe_itanium_class (insn) == ITANIUM_CLASS_IALU)
7357       {
7358         sd_iterator_def sd_it;
7359         dep_t dep;
7360         bool has_mem_op_consumer_p = false;
7361
7362         FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
7363           {
7364             enum attr_itanium_class c;
7365
7366             if (DEP_TYPE (dep) != REG_DEP_TRUE)
7367               continue;
7368
7369             next = DEP_CON (dep);
7370             c = ia64_safe_itanium_class (next);
7371             if ((c == ITANIUM_CLASS_ST
7372                  || c == ITANIUM_CLASS_STF)
7373                 && ia64_st_address_bypass_p (insn, next))
7374               {
7375                 has_mem_op_consumer_p = true;
7376                 break;
7377               }
7378             else if ((c == ITANIUM_CLASS_LD
7379                       || c == ITANIUM_CLASS_FLD
7380                       || c == ITANIUM_CLASS_FLDP)
7381                      && ia64_ld_address_bypass_p (insn, next))
7382               {
7383                 has_mem_op_consumer_p = true;
7384                 break;
7385               }
7386           }
7387
7388         insn->call = has_mem_op_consumer_p;
7389       }
7390 }
7391
7392 /* We're beginning a new block.  Initialize data structures as necessary.  */
7393
7394 static void
7395 ia64_sched_init (FILE *dump ATTRIBUTE_UNUSED,
7396                  int sched_verbose ATTRIBUTE_UNUSED,
7397                  int max_ready ATTRIBUTE_UNUSED)
7398 {
7399   if (flag_checking && !sel_sched_p () && reload_completed)
7400     {
7401       for (rtx_insn *insn = NEXT_INSN (current_sched_info->prev_head);
7402            insn != current_sched_info->next_tail;
7403            insn = NEXT_INSN (insn))
7404         gcc_assert (!SCHED_GROUP_P (insn));
7405     }
7406   last_scheduled_insn = NULL;
7407   init_insn_group_barriers ();
7408
7409   current_cycle = 0;
7410   memset (mem_ops_in_group, 0, sizeof (mem_ops_in_group));
7411 }
7412
7413 /* We're beginning a scheduling pass.  Check assertion.  */
7414
7415 static void
7416 ia64_sched_init_global (FILE *dump ATTRIBUTE_UNUSED,
7417                         int sched_verbose ATTRIBUTE_UNUSED,
7418                         int max_ready ATTRIBUTE_UNUSED)
7419 {
7420   gcc_assert (pending_data_specs == 0);
7421 }
7422
7423 /* Scheduling pass is now finished.  Free/reset static variable.  */
7424 static void
7425 ia64_sched_finish_global (FILE *dump ATTRIBUTE_UNUSED,
7426                           int sched_verbose ATTRIBUTE_UNUSED)
7427 {
7428   gcc_assert (pending_data_specs == 0);
7429 }
7430
7431 /* Return TRUE if INSN is a load (either normal or speculative, but not a
7432    speculation check), FALSE otherwise.  */
7433 static bool
7434 is_load_p (rtx_insn *insn)
7435 {
7436   enum attr_itanium_class insn_class = ia64_safe_itanium_class (insn);
7437
7438   return
7439    ((insn_class == ITANIUM_CLASS_LD || insn_class == ITANIUM_CLASS_FLD)
7440     && get_attr_check_load (insn) == CHECK_LOAD_NO);
7441 }
7442
7443 /* If INSN is a memory reference, memoize it in MEM_OPS_IN_GROUP global array
7444    (taking account for 3-cycle cache reference postponing for stores: Intel
7445    Itanium 2 Reference Manual for Software Development and Optimization,
7446    6.7.3.1).  */
7447 static void
7448 record_memory_reference (rtx_insn *insn)
7449 {
7450   enum attr_itanium_class insn_class = ia64_safe_itanium_class (insn);
7451
7452   switch (insn_class) {
7453     case ITANIUM_CLASS_FLD:
7454     case ITANIUM_CLASS_LD:
7455       mem_ops_in_group[current_cycle % 4]++;
7456       break;
7457     case ITANIUM_CLASS_STF:
7458     case ITANIUM_CLASS_ST:
7459       mem_ops_in_group[(current_cycle + 3) % 4]++;
7460       break;
7461     default:;
7462   }
7463 }
7464
7465 /* We are about to being issuing insns for this clock cycle.
7466    Override the default sort algorithm to better slot instructions.  */
7467
7468 static int
7469 ia64_dfa_sched_reorder (FILE *dump, int sched_verbose, rtx_insn **ready,
7470                         int *pn_ready, int clock_var,
7471                         int reorder_type)
7472 {
7473   int n_asms;
7474   int n_ready = *pn_ready;
7475   rtx_insn **e_ready = ready + n_ready;
7476   rtx_insn **insnp;
7477
7478   if (sched_verbose)
7479     fprintf (dump, "// ia64_dfa_sched_reorder (type %d):\n", reorder_type);
7480
7481   if (reorder_type == 0)
7482     {
7483       /* First, move all USEs, CLOBBERs and other crud out of the way.  */
7484       n_asms = 0;
7485       for (insnp = ready; insnp < e_ready; insnp++)
7486         if (insnp < e_ready)
7487           {
7488             rtx_insn *insn = *insnp;
7489             enum attr_type t = ia64_safe_type (insn);
7490             if (t == TYPE_UNKNOWN)
7491               {
7492                 if (GET_CODE (PATTERN (insn)) == ASM_INPUT
7493                     || asm_noperands (PATTERN (insn)) >= 0)
7494                   {
7495                     rtx_insn *lowest = ready[n_asms];
7496                     ready[n_asms] = insn;
7497                     *insnp = lowest;
7498                     n_asms++;
7499                   }
7500                 else
7501                   {
7502                     rtx_insn *highest = ready[n_ready - 1];
7503                     ready[n_ready - 1] = insn;
7504                     *insnp = highest;
7505                     return 1;
7506                   }
7507               }
7508           }
7509
7510       if (n_asms < n_ready)
7511         {
7512           /* Some normal insns to process.  Skip the asms.  */
7513           ready += n_asms;
7514           n_ready -= n_asms;
7515         }
7516       else if (n_ready > 0)
7517         return 1;
7518     }
7519
7520   if (ia64_final_schedule)
7521     {
7522       int deleted = 0;
7523       int nr_need_stop = 0;
7524
7525       for (insnp = ready; insnp < e_ready; insnp++)
7526         if (safe_group_barrier_needed (*insnp))
7527           nr_need_stop++;
7528
7529       if (reorder_type == 1 && n_ready == nr_need_stop)
7530         return 0;
7531       if (reorder_type == 0)
7532         return 1;
7533       insnp = e_ready;
7534       /* Move down everything that needs a stop bit, preserving
7535          relative order.  */
7536       while (insnp-- > ready + deleted)
7537         while (insnp >= ready + deleted)
7538           {
7539             rtx_insn *insn = *insnp;
7540             if (! safe_group_barrier_needed (insn))
7541               break;
7542             memmove (ready + 1, ready, (insnp - ready) * sizeof (rtx));
7543             *ready = insn;
7544             deleted++;
7545           }
7546       n_ready -= deleted;
7547       ready += deleted;
7548     }
7549
7550   current_cycle = clock_var;
7551   if (reload_completed && mem_ops_in_group[clock_var % 4] >= ia64_max_memory_insns)
7552     {
7553       int moved = 0;
7554
7555       insnp = e_ready;
7556       /* Move down loads/stores, preserving relative order.  */
7557       while (insnp-- > ready + moved)
7558         while (insnp >= ready + moved)
7559           {
7560             rtx_insn *insn = *insnp;
7561             if (! is_load_p (insn))
7562               break;
7563             memmove (ready + 1, ready, (insnp - ready) * sizeof (rtx));
7564             *ready = insn;
7565             moved++;
7566           }
7567       n_ready -= moved;
7568       ready += moved;
7569     }
7570
7571   return 1;
7572 }
7573
7574 /* We are about to being issuing insns for this clock cycle.  Override
7575    the default sort algorithm to better slot instructions.  */
7576
7577 static int
7578 ia64_sched_reorder (FILE *dump, int sched_verbose, rtx_insn **ready,
7579                     int *pn_ready, int clock_var)
7580 {
7581   return ia64_dfa_sched_reorder (dump, sched_verbose, ready,
7582                                  pn_ready, clock_var, 0);
7583 }
7584
7585 /* Like ia64_sched_reorder, but called after issuing each insn.
7586    Override the default sort algorithm to better slot instructions.  */
7587
7588 static int
7589 ia64_sched_reorder2 (FILE *dump ATTRIBUTE_UNUSED,
7590                      int sched_verbose ATTRIBUTE_UNUSED, rtx_insn **ready,
7591                      int *pn_ready, int clock_var)
7592 {
7593   return ia64_dfa_sched_reorder (dump, sched_verbose, ready, pn_ready,
7594                                  clock_var, 1);
7595 }
7596
7597 /* We are about to issue INSN.  Return the number of insns left on the
7598    ready queue that can be issued this cycle.  */
7599
7600 static int
7601 ia64_variable_issue (FILE *dump ATTRIBUTE_UNUSED,
7602                      int sched_verbose ATTRIBUTE_UNUSED,
7603                      rtx_insn *insn,
7604                      int can_issue_more ATTRIBUTE_UNUSED)
7605 {
7606   if (sched_deps_info->generate_spec_deps && !sel_sched_p ())
7607     /* Modulo scheduling does not extend h_i_d when emitting
7608        new instructions.  Don't use h_i_d, if we don't have to.  */
7609     {
7610       if (DONE_SPEC (insn) & BEGIN_DATA)
7611         pending_data_specs++;
7612       if (CHECK_SPEC (insn) & BEGIN_DATA)
7613         pending_data_specs--;
7614     }
7615
7616   if (DEBUG_INSN_P (insn))
7617     return 1;
7618
7619   last_scheduled_insn = insn;
7620   memcpy (prev_cycle_state, curr_state, dfa_state_size);
7621   if (reload_completed)
7622     {
7623       int needed = group_barrier_needed (insn);
7624
7625       gcc_assert (!needed);
7626       if (CALL_P (insn))
7627         init_insn_group_barriers ();
7628       stops_p [INSN_UID (insn)] = stop_before_p;
7629       stop_before_p = 0;
7630
7631       record_memory_reference (insn);
7632     }
7633   return 1;
7634 }
7635
7636 /* We are choosing insn from the ready queue.  Return zero if INSN
7637    can be chosen.  */
7638
7639 static int
7640 ia64_first_cycle_multipass_dfa_lookahead_guard (rtx_insn *insn, int ready_index)
7641 {
7642   gcc_assert (insn && INSN_P (insn));
7643
7644   /* Size of ALAT is 32.  As far as we perform conservative
7645      data speculation, we keep ALAT half-empty.  */
7646   if (pending_data_specs >= 16 && (TODO_SPEC (insn) & BEGIN_DATA))
7647     return ready_index == 0 ? -1 : 1;
7648
7649   if (ready_index == 0)
7650     return 0;
7651
7652   if ((!reload_completed
7653        || !safe_group_barrier_needed (insn))
7654       && (!mflag_sched_mem_insns_hard_limit
7655           || !is_load_p (insn)
7656           || mem_ops_in_group[current_cycle % 4] < ia64_max_memory_insns))
7657     return 0;
7658
7659   return 1;
7660 }
7661
7662 /* The following variable value is pseudo-insn used by the DFA insn
7663    scheduler to change the DFA state when the simulated clock is
7664    increased.  */
7665
7666 static rtx_insn *dfa_pre_cycle_insn;
7667
7668 /* Returns 1 when a meaningful insn was scheduled between the last group
7669    barrier and LAST.  */
7670 static int
7671 scheduled_good_insn (rtx_insn *last)
7672 {
7673   if (last && recog_memoized (last) >= 0)
7674     return 1;
7675
7676   for ( ;
7677        last != NULL && !NOTE_INSN_BASIC_BLOCK_P (last)
7678        && !stops_p[INSN_UID (last)];
7679        last = PREV_INSN (last))
7680     /* We could hit a NOTE_INSN_DELETED here which is actually outside
7681        the ebb we're scheduling.  */
7682     if (INSN_P (last) && recog_memoized (last) >= 0)
7683       return 1;
7684
7685   return 0;
7686 }
7687
7688 /* We are about to being issuing INSN.  Return nonzero if we cannot
7689    issue it on given cycle CLOCK and return zero if we should not sort
7690    the ready queue on the next clock start.  */
7691
7692 static int
7693 ia64_dfa_new_cycle (FILE *dump, int verbose, rtx_insn *insn, int last_clock,
7694                     int clock, int *sort_p)
7695 {
7696   gcc_assert (insn && INSN_P (insn));
7697
7698   if (DEBUG_INSN_P (insn))
7699     return 0;
7700
7701   /* When a group barrier is needed for insn, last_scheduled_insn
7702      should be set.  */
7703   gcc_assert (!(reload_completed && safe_group_barrier_needed (insn))
7704               || last_scheduled_insn);
7705
7706   if ((reload_completed
7707        && (safe_group_barrier_needed (insn)
7708            || (mflag_sched_stop_bits_after_every_cycle
7709                && last_clock != clock
7710                && last_scheduled_insn
7711                && scheduled_good_insn (last_scheduled_insn))))
7712       || (last_scheduled_insn
7713           && (CALL_P (last_scheduled_insn)
7714               || unknown_for_bundling_p (last_scheduled_insn))))
7715     {
7716       init_insn_group_barriers ();
7717
7718       if (verbose && dump)
7719         fprintf (dump, "//    Stop should be before %d%s\n", INSN_UID (insn),
7720                  last_clock == clock ? " + cycle advance" : "");
7721
7722       stop_before_p = 1;
7723       current_cycle = clock;
7724       mem_ops_in_group[current_cycle % 4] = 0;
7725
7726       if (last_clock == clock)
7727         {
7728           state_transition (curr_state, dfa_stop_insn);
7729           if (TARGET_EARLY_STOP_BITS)
7730             *sort_p = (last_scheduled_insn == NULL_RTX
7731                        || ! CALL_P (last_scheduled_insn));
7732           else
7733             *sort_p = 0;
7734           return 1;
7735         }
7736
7737       if (last_scheduled_insn)
7738         {
7739           if (unknown_for_bundling_p (last_scheduled_insn))
7740             state_reset (curr_state);
7741           else
7742             {
7743               memcpy (curr_state, prev_cycle_state, dfa_state_size);
7744               state_transition (curr_state, dfa_stop_insn);
7745               state_transition (curr_state, dfa_pre_cycle_insn);
7746               state_transition (curr_state, NULL);
7747             }
7748         }
7749     }
7750   return 0;
7751 }
7752
7753 /* Implement targetm.sched.h_i_d_extended hook.
7754    Extend internal data structures.  */
7755 static void
7756 ia64_h_i_d_extended (void)
7757 {
7758   if (stops_p != NULL)
7759     {
7760       int new_clocks_length = get_max_uid () * 3 / 2;
7761       stops_p = (char *) xrecalloc (stops_p, new_clocks_length, clocks_length, 1);
7762       clocks_length = new_clocks_length;
7763     }
7764 }
7765 \f
7766
7767 /* This structure describes the data used by the backend to guide scheduling.
7768    When the current scheduling point is switched, this data should be saved
7769    and restored later, if the scheduler returns to this point.  */
7770 struct _ia64_sched_context
7771 {
7772   state_t prev_cycle_state;
7773   rtx_insn *last_scheduled_insn;
7774   struct reg_write_state rws_sum[NUM_REGS];
7775   struct reg_write_state rws_insn[NUM_REGS];
7776   int first_instruction;
7777   int pending_data_specs;
7778   int current_cycle;
7779   char mem_ops_in_group[4];
7780 };
7781 typedef struct _ia64_sched_context *ia64_sched_context_t;
7782
7783 /* Allocates a scheduling context.  */
7784 static void *
7785 ia64_alloc_sched_context (void)
7786 {
7787   return xmalloc (sizeof (struct _ia64_sched_context));
7788 }
7789
7790 /* Initializes the _SC context with clean data, if CLEAN_P, and from
7791    the global context otherwise.  */
7792 static void
7793 ia64_init_sched_context (void *_sc, bool clean_p)
7794 {
7795   ia64_sched_context_t sc = (ia64_sched_context_t) _sc;
7796
7797   sc->prev_cycle_state = xmalloc (dfa_state_size);
7798   if (clean_p)
7799     {
7800       state_reset (sc->prev_cycle_state);
7801       sc->last_scheduled_insn = NULL;
7802       memset (sc->rws_sum, 0, sizeof (rws_sum));
7803       memset (sc->rws_insn, 0, sizeof (rws_insn));
7804       sc->first_instruction = 1;
7805       sc->pending_data_specs = 0;
7806       sc->current_cycle = 0;
7807       memset (sc->mem_ops_in_group, 0, sizeof (mem_ops_in_group));
7808     }
7809   else
7810     {
7811       memcpy (sc->prev_cycle_state, prev_cycle_state, dfa_state_size);
7812       sc->last_scheduled_insn = last_scheduled_insn;
7813       memcpy (sc->rws_sum, rws_sum, sizeof (rws_sum));
7814       memcpy (sc->rws_insn, rws_insn, sizeof (rws_insn));
7815       sc->first_instruction = first_instruction;
7816       sc->pending_data_specs = pending_data_specs;
7817       sc->current_cycle = current_cycle;
7818       memcpy (sc->mem_ops_in_group, mem_ops_in_group, sizeof (mem_ops_in_group));
7819     }
7820 }
7821
7822 /* Sets the global scheduling context to the one pointed to by _SC.  */
7823 static void
7824 ia64_set_sched_context (void *_sc)
7825 {
7826   ia64_sched_context_t sc = (ia64_sched_context_t) _sc;
7827
7828   gcc_assert (sc != NULL);
7829
7830   memcpy (prev_cycle_state, sc->prev_cycle_state, dfa_state_size);
7831   last_scheduled_insn = sc->last_scheduled_insn;
7832   memcpy (rws_sum, sc->rws_sum, sizeof (rws_sum));
7833   memcpy (rws_insn, sc->rws_insn, sizeof (rws_insn));
7834   first_instruction = sc->first_instruction;
7835   pending_data_specs = sc->pending_data_specs;
7836   current_cycle = sc->current_cycle;
7837   memcpy (mem_ops_in_group, sc->mem_ops_in_group, sizeof (mem_ops_in_group));
7838 }
7839
7840 /* Clears the data in the _SC scheduling context.  */
7841 static void
7842 ia64_clear_sched_context (void *_sc)
7843 {
7844   ia64_sched_context_t sc = (ia64_sched_context_t) _sc;
7845
7846   free (sc->prev_cycle_state);
7847   sc->prev_cycle_state = NULL;
7848 }
7849
7850 /* Frees the _SC scheduling context.  */
7851 static void
7852 ia64_free_sched_context (void *_sc)
7853 {
7854   gcc_assert (_sc != NULL);
7855
7856   free (_sc);
7857 }
7858
7859 typedef rtx (* gen_func_t) (rtx, rtx);
7860
7861 /* Return a function that will generate a load of mode MODE_NO
7862    with speculation types TS.  */
7863 static gen_func_t
7864 get_spec_load_gen_function (ds_t ts, int mode_no)
7865 {
7866   static gen_func_t gen_ld_[] = {
7867     gen_movbi,
7868     gen_movqi_internal,
7869     gen_movhi_internal,
7870     gen_movsi_internal,
7871     gen_movdi_internal,
7872     gen_movsf_internal,
7873     gen_movdf_internal,
7874     gen_movxf_internal,
7875     gen_movti_internal,
7876     gen_zero_extendqidi2,
7877     gen_zero_extendhidi2,
7878     gen_zero_extendsidi2,
7879   };
7880
7881   static gen_func_t gen_ld_a[] = {
7882     gen_movbi_advanced,
7883     gen_movqi_advanced,
7884     gen_movhi_advanced,
7885     gen_movsi_advanced,
7886     gen_movdi_advanced,
7887     gen_movsf_advanced,
7888     gen_movdf_advanced,
7889     gen_movxf_advanced,
7890     gen_movti_advanced,
7891     gen_zero_extendqidi2_advanced,
7892     gen_zero_extendhidi2_advanced,
7893     gen_zero_extendsidi2_advanced,
7894   };
7895   static gen_func_t gen_ld_s[] = {
7896     gen_movbi_speculative,
7897     gen_movqi_speculative,
7898     gen_movhi_speculative,
7899     gen_movsi_speculative,
7900     gen_movdi_speculative,
7901     gen_movsf_speculative,
7902     gen_movdf_speculative,
7903     gen_movxf_speculative,
7904     gen_movti_speculative,
7905     gen_zero_extendqidi2_speculative,
7906     gen_zero_extendhidi2_speculative,
7907     gen_zero_extendsidi2_speculative,
7908   };
7909   static gen_func_t gen_ld_sa[] = {
7910     gen_movbi_speculative_advanced,
7911     gen_movqi_speculative_advanced,
7912     gen_movhi_speculative_advanced,
7913     gen_movsi_speculative_advanced,
7914     gen_movdi_speculative_advanced,
7915     gen_movsf_speculative_advanced,
7916     gen_movdf_speculative_advanced,
7917     gen_movxf_speculative_advanced,
7918     gen_movti_speculative_advanced,
7919     gen_zero_extendqidi2_speculative_advanced,
7920     gen_zero_extendhidi2_speculative_advanced,
7921     gen_zero_extendsidi2_speculative_advanced,
7922   };
7923   static gen_func_t gen_ld_s_a[] = {
7924     gen_movbi_speculative_a,
7925     gen_movqi_speculative_a,
7926     gen_movhi_speculative_a,
7927     gen_movsi_speculative_a,
7928     gen_movdi_speculative_a,
7929     gen_movsf_speculative_a,
7930     gen_movdf_speculative_a,
7931     gen_movxf_speculative_a,
7932     gen_movti_speculative_a,
7933     gen_zero_extendqidi2_speculative_a,
7934     gen_zero_extendhidi2_speculative_a,
7935     gen_zero_extendsidi2_speculative_a,
7936   };
7937
7938   gen_func_t *gen_ld;
7939
7940   if (ts & BEGIN_DATA)
7941     {
7942       if (ts & BEGIN_CONTROL)
7943         gen_ld = gen_ld_sa;
7944       else
7945         gen_ld = gen_ld_a;
7946     }
7947   else if (ts & BEGIN_CONTROL)
7948     {
7949       if ((spec_info->flags & SEL_SCHED_SPEC_DONT_CHECK_CONTROL)
7950           || ia64_needs_block_p (ts))
7951         gen_ld = gen_ld_s;
7952       else
7953         gen_ld = gen_ld_s_a;
7954     }
7955   else if (ts == 0)
7956     gen_ld = gen_ld_;
7957   else
7958     gcc_unreachable ();
7959
7960   return gen_ld[mode_no];
7961 }
7962
7963 /* Constants that help mapping 'machine_mode' to int.  */
7964 enum SPEC_MODES
7965   {
7966     SPEC_MODE_INVALID = -1,
7967     SPEC_MODE_FIRST = 0,
7968     SPEC_MODE_FOR_EXTEND_FIRST = 1,
7969     SPEC_MODE_FOR_EXTEND_LAST = 3,
7970     SPEC_MODE_LAST = 8
7971   };
7972
7973 enum
7974   {
7975     /* Offset to reach ZERO_EXTEND patterns.  */
7976     SPEC_GEN_EXTEND_OFFSET = SPEC_MODE_LAST - SPEC_MODE_FOR_EXTEND_FIRST + 1
7977   };
7978
7979 /* Return index of the MODE.  */
7980 static int
7981 ia64_mode_to_int (machine_mode mode)
7982 {
7983   switch (mode)
7984     {
7985     case E_BImode: return 0; /* SPEC_MODE_FIRST  */
7986     case E_QImode: return 1; /* SPEC_MODE_FOR_EXTEND_FIRST  */
7987     case E_HImode: return 2;
7988     case E_SImode: return 3; /* SPEC_MODE_FOR_EXTEND_LAST  */
7989     case E_DImode: return 4;
7990     case E_SFmode: return 5;
7991     case E_DFmode: return 6;
7992     case E_XFmode: return 7;
7993     case E_TImode:
7994       /* ??? This mode needs testing.  Bypasses for ldfp8 instruction are not
7995          mentioned in itanium[12].md.  Predicate fp_register_operand also
7996          needs to be defined.  Bottom line: better disable for now.  */
7997       return SPEC_MODE_INVALID;
7998     default:     return SPEC_MODE_INVALID;
7999     }
8000 }
8001
8002 /* Provide information about speculation capabilities.  */
8003 static void
8004 ia64_set_sched_flags (spec_info_t spec_info)
8005 {
8006   unsigned int *flags = &(current_sched_info->flags);
8007
8008   if (*flags & SCHED_RGN
8009       || *flags & SCHED_EBB
8010       || *flags & SEL_SCHED)
8011     {
8012       int mask = 0;
8013
8014       if ((mflag_sched_br_data_spec && !reload_completed && optimize > 0)
8015           || (mflag_sched_ar_data_spec && reload_completed))
8016         {
8017           mask |= BEGIN_DATA;
8018
8019           if (!sel_sched_p ()
8020               && ((mflag_sched_br_in_data_spec && !reload_completed)
8021                   || (mflag_sched_ar_in_data_spec && reload_completed)))
8022             mask |= BE_IN_DATA;
8023         }
8024
8025       if (mflag_sched_control_spec
8026           && (!sel_sched_p ()
8027               || reload_completed))
8028         {
8029           mask |= BEGIN_CONTROL;
8030
8031           if (!sel_sched_p () && mflag_sched_in_control_spec)
8032             mask |= BE_IN_CONTROL;
8033         }
8034
8035       spec_info->mask = mask;
8036
8037       if (mask)
8038         {
8039           *flags |= USE_DEPS_LIST | DO_SPECULATION;
8040
8041           if (mask & BE_IN_SPEC)
8042             *flags |= NEW_BBS;
8043
8044           spec_info->flags = 0;
8045
8046           if ((mask & CONTROL_SPEC)
8047               && sel_sched_p () && mflag_sel_sched_dont_check_control_spec)
8048             spec_info->flags |= SEL_SCHED_SPEC_DONT_CHECK_CONTROL;
8049
8050           if (sched_verbose >= 1)
8051             spec_info->dump = sched_dump;
8052           else
8053             spec_info->dump = 0;
8054
8055           if (mflag_sched_count_spec_in_critical_path)
8056             spec_info->flags |= COUNT_SPEC_IN_CRITICAL_PATH;
8057         }
8058     }
8059   else
8060     spec_info->mask = 0;
8061 }
8062
8063 /* If INSN is an appropriate load return its mode.
8064    Return -1 otherwise.  */
8065 static int
8066 get_mode_no_for_insn (rtx_insn *insn)
8067 {
8068   rtx reg, mem, mode_rtx;
8069   int mode_no;
8070   bool extend_p;
8071
8072   extract_insn_cached (insn);
8073
8074   /* We use WHICH_ALTERNATIVE only after reload.  This will
8075      guarantee that reload won't touch a speculative insn.  */
8076
8077   if (recog_data.n_operands != 2)
8078     return -1;
8079
8080   reg = recog_data.operand[0];
8081   mem = recog_data.operand[1];
8082
8083   /* We should use MEM's mode since REG's mode in presence of
8084      ZERO_EXTEND will always be DImode.  */
8085   if (get_attr_speculable1 (insn) == SPECULABLE1_YES)
8086     /* Process non-speculative ld.  */
8087     {
8088       if (!reload_completed)
8089         {
8090           /* Do not speculate into regs like ar.lc.  */
8091           if (!REG_P (reg) || AR_REGNO_P (REGNO (reg)))
8092             return -1;
8093
8094           if (!MEM_P (mem))
8095             return -1;
8096
8097           {
8098             rtx mem_reg = XEXP (mem, 0);
8099
8100             if (!REG_P (mem_reg))
8101               return -1;
8102           }
8103
8104           mode_rtx = mem;
8105         }
8106       else if (get_attr_speculable2 (insn) == SPECULABLE2_YES)
8107         {
8108           gcc_assert (REG_P (reg) && MEM_P (mem));
8109           mode_rtx = mem;
8110         }
8111       else
8112         return -1;
8113     }
8114   else if (get_attr_data_speculative (insn) == DATA_SPECULATIVE_YES
8115            || get_attr_control_speculative (insn) == CONTROL_SPECULATIVE_YES
8116            || get_attr_check_load (insn) == CHECK_LOAD_YES)
8117     /* Process speculative ld or ld.c.  */
8118     {
8119       gcc_assert (REG_P (reg) && MEM_P (mem));
8120       mode_rtx = mem;
8121     }
8122   else
8123     {
8124       enum attr_itanium_class attr_class = get_attr_itanium_class (insn);
8125
8126       if (attr_class == ITANIUM_CLASS_CHK_A
8127           || attr_class == ITANIUM_CLASS_CHK_S_I
8128           || attr_class == ITANIUM_CLASS_CHK_S_F)
8129         /* Process chk.  */
8130         mode_rtx = reg;
8131       else
8132         return -1;
8133     }
8134
8135   mode_no = ia64_mode_to_int (GET_MODE (mode_rtx));
8136
8137   if (mode_no == SPEC_MODE_INVALID)
8138     return -1;
8139
8140   extend_p = (GET_MODE (reg) != GET_MODE (mode_rtx));
8141
8142   if (extend_p)
8143     {
8144       if (!(SPEC_MODE_FOR_EXTEND_FIRST <= mode_no
8145             && mode_no <= SPEC_MODE_FOR_EXTEND_LAST))
8146         return -1;
8147
8148       mode_no += SPEC_GEN_EXTEND_OFFSET;
8149     }
8150
8151   return mode_no;
8152 }
8153
8154 /* If X is an unspec part of a speculative load, return its code.
8155    Return -1 otherwise.  */
8156 static int
8157 get_spec_unspec_code (const_rtx x)
8158 {
8159   if (GET_CODE (x) != UNSPEC)
8160     return -1;
8161
8162   {
8163     int code;
8164
8165     code = XINT (x, 1);
8166
8167     switch (code)
8168       {
8169       case UNSPEC_LDA:
8170       case UNSPEC_LDS:
8171       case UNSPEC_LDS_A:
8172       case UNSPEC_LDSA:
8173         return code;
8174
8175       default:
8176         return -1;
8177       }
8178   }
8179 }
8180
8181 /* Implement skip_rtx_p hook.  */
8182 static bool
8183 ia64_skip_rtx_p (const_rtx x)
8184 {
8185   return get_spec_unspec_code (x) != -1;
8186 }
8187
8188 /* If INSN is a speculative load, return its UNSPEC code.
8189    Return -1 otherwise.  */
8190 static int
8191 get_insn_spec_code (const_rtx insn)
8192 {
8193   rtx pat, reg, mem;
8194
8195   pat = PATTERN (insn);
8196
8197   if (GET_CODE (pat) == COND_EXEC)
8198     pat = COND_EXEC_CODE (pat);
8199
8200   if (GET_CODE (pat) != SET)
8201     return -1;
8202
8203   reg = SET_DEST (pat);
8204   if (!REG_P (reg))
8205     return -1;
8206
8207   mem = SET_SRC (pat);
8208   if (GET_CODE (mem) == ZERO_EXTEND)
8209     mem = XEXP (mem, 0);
8210
8211   return get_spec_unspec_code (mem);
8212 }
8213
8214 /* If INSN is a speculative load, return a ds with the speculation types.
8215    Otherwise [if INSN is a normal instruction] return 0.  */
8216 static ds_t
8217 ia64_get_insn_spec_ds (rtx_insn *insn)
8218 {
8219   int code = get_insn_spec_code (insn);
8220
8221   switch (code)
8222     {
8223     case UNSPEC_LDA:
8224       return BEGIN_DATA;
8225
8226     case UNSPEC_LDS:
8227     case UNSPEC_LDS_A:
8228       return BEGIN_CONTROL;
8229
8230     case UNSPEC_LDSA:
8231       return BEGIN_DATA | BEGIN_CONTROL;
8232
8233     default:
8234       return 0;
8235     }
8236 }
8237
8238 /* If INSN is a speculative load return a ds with the speculation types that
8239    will be checked.
8240    Otherwise [if INSN is a normal instruction] return 0.  */
8241 static ds_t
8242 ia64_get_insn_checked_ds (rtx_insn *insn)
8243 {
8244   int code = get_insn_spec_code (insn);
8245
8246   switch (code)
8247     {
8248     case UNSPEC_LDA:
8249       return BEGIN_DATA | BEGIN_CONTROL;
8250
8251     case UNSPEC_LDS:
8252       return BEGIN_CONTROL;
8253
8254     case UNSPEC_LDS_A:
8255     case UNSPEC_LDSA:
8256       return BEGIN_DATA | BEGIN_CONTROL;
8257
8258     default:
8259       return 0;
8260     }
8261 }
8262
8263 /* If GEN_P is true, calculate the index of needed speculation check and return
8264    speculative pattern for INSN with speculative mode TS, machine mode
8265    MODE_NO and with ZERO_EXTEND (if EXTEND_P is true).
8266    If GEN_P is false, just calculate the index of needed speculation check.  */
8267 static rtx
8268 ia64_gen_spec_load (rtx insn, ds_t ts, int mode_no)
8269 {
8270   rtx pat, new_pat;
8271   gen_func_t gen_load;
8272
8273   gen_load = get_spec_load_gen_function (ts, mode_no);
8274
8275   new_pat = gen_load (copy_rtx (recog_data.operand[0]),
8276                       copy_rtx (recog_data.operand[1]));
8277
8278   pat = PATTERN (insn);
8279   if (GET_CODE (pat) == COND_EXEC)
8280     new_pat = gen_rtx_COND_EXEC (VOIDmode, copy_rtx (COND_EXEC_TEST (pat)),
8281                                  new_pat);
8282
8283   return new_pat;
8284 }
8285
8286 static bool
8287 insn_can_be_in_speculative_p (rtx insn ATTRIBUTE_UNUSED,
8288                               ds_t ds ATTRIBUTE_UNUSED)
8289 {
8290   return false;
8291 }
8292
8293 /* Implement targetm.sched.speculate_insn hook.
8294    Check if the INSN can be TS speculative.
8295    If 'no' - return -1.
8296    If 'yes' - generate speculative pattern in the NEW_PAT and return 1.
8297    If current pattern of the INSN already provides TS speculation,
8298    return 0.  */
8299 static int
8300 ia64_speculate_insn (rtx_insn *insn, ds_t ts, rtx *new_pat)
8301 {
8302   int mode_no;
8303   int res;
8304
8305   gcc_assert (!(ts & ~SPECULATIVE));
8306
8307   if (ia64_spec_check_p (insn))
8308     return -1;
8309
8310   if ((ts & BE_IN_SPEC)
8311       && !insn_can_be_in_speculative_p (insn, ts))
8312     return -1;
8313
8314   mode_no = get_mode_no_for_insn (insn);
8315
8316   if (mode_no != SPEC_MODE_INVALID)
8317     {
8318       if (ia64_get_insn_spec_ds (insn) == ds_get_speculation_types (ts))
8319         res = 0;
8320       else
8321         {
8322           res = 1;
8323           *new_pat = ia64_gen_spec_load (insn, ts, mode_no);
8324         }
8325     }
8326   else
8327     res = -1;
8328
8329   return res;
8330 }
8331
8332 /* Return a function that will generate a check for speculation TS with mode
8333    MODE_NO.
8334    If simple check is needed, pass true for SIMPLE_CHECK_P.
8335    If clearing check is needed, pass true for CLEARING_CHECK_P.  */
8336 static gen_func_t
8337 get_spec_check_gen_function (ds_t ts, int mode_no,
8338                              bool simple_check_p, bool clearing_check_p)
8339 {
8340   static gen_func_t gen_ld_c_clr[] = {
8341     gen_movbi_clr,
8342     gen_movqi_clr,
8343     gen_movhi_clr,
8344     gen_movsi_clr,
8345     gen_movdi_clr,
8346     gen_movsf_clr,
8347     gen_movdf_clr,
8348     gen_movxf_clr,
8349     gen_movti_clr,
8350     gen_zero_extendqidi2_clr,
8351     gen_zero_extendhidi2_clr,
8352     gen_zero_extendsidi2_clr,
8353   };
8354   static gen_func_t gen_ld_c_nc[] = {
8355     gen_movbi_nc,
8356     gen_movqi_nc,
8357     gen_movhi_nc,
8358     gen_movsi_nc,
8359     gen_movdi_nc,
8360     gen_movsf_nc,
8361     gen_movdf_nc,
8362     gen_movxf_nc,
8363     gen_movti_nc,
8364     gen_zero_extendqidi2_nc,
8365     gen_zero_extendhidi2_nc,
8366     gen_zero_extendsidi2_nc,
8367   };
8368   static gen_func_t gen_chk_a_clr[] = {
8369     gen_advanced_load_check_clr_bi,
8370     gen_advanced_load_check_clr_qi,
8371     gen_advanced_load_check_clr_hi,
8372     gen_advanced_load_check_clr_si,
8373     gen_advanced_load_check_clr_di,
8374     gen_advanced_load_check_clr_sf,
8375     gen_advanced_load_check_clr_df,
8376     gen_advanced_load_check_clr_xf,
8377     gen_advanced_load_check_clr_ti,
8378     gen_advanced_load_check_clr_di,
8379     gen_advanced_load_check_clr_di,
8380     gen_advanced_load_check_clr_di,
8381   };
8382   static gen_func_t gen_chk_a_nc[] = {
8383     gen_advanced_load_check_nc_bi,
8384     gen_advanced_load_check_nc_qi,
8385     gen_advanced_load_check_nc_hi,
8386     gen_advanced_load_check_nc_si,
8387     gen_advanced_load_check_nc_di,
8388     gen_advanced_load_check_nc_sf,
8389     gen_advanced_load_check_nc_df,
8390     gen_advanced_load_check_nc_xf,
8391     gen_advanced_load_check_nc_ti,
8392     gen_advanced_load_check_nc_di,
8393     gen_advanced_load_check_nc_di,
8394     gen_advanced_load_check_nc_di,
8395   };
8396   static gen_func_t gen_chk_s[] = {
8397     gen_speculation_check_bi,
8398     gen_speculation_check_qi,
8399     gen_speculation_check_hi,
8400     gen_speculation_check_si,
8401     gen_speculation_check_di,
8402     gen_speculation_check_sf,
8403     gen_speculation_check_df,
8404     gen_speculation_check_xf,
8405     gen_speculation_check_ti,
8406     gen_speculation_check_di,
8407     gen_speculation_check_di,
8408     gen_speculation_check_di,
8409   };
8410
8411   gen_func_t *gen_check;
8412
8413   if (ts & BEGIN_DATA)
8414     {
8415       /* We don't need recovery because even if this is ld.sa
8416          ALAT entry will be allocated only if NAT bit is set to zero.
8417          So it is enough to use ld.c here.  */
8418
8419       if (simple_check_p)
8420         {
8421           gcc_assert (mflag_sched_spec_ldc);
8422
8423           if (clearing_check_p)
8424             gen_check = gen_ld_c_clr;
8425           else
8426             gen_check = gen_ld_c_nc;
8427         }
8428       else
8429         {
8430           if (clearing_check_p)
8431             gen_check = gen_chk_a_clr;
8432           else
8433             gen_check = gen_chk_a_nc;
8434         }
8435     }
8436   else if (ts & BEGIN_CONTROL)
8437     {
8438       if (simple_check_p)
8439         /* We might want to use ld.sa -> ld.c instead of
8440            ld.s -> chk.s.  */
8441         {
8442           gcc_assert (!ia64_needs_block_p (ts));
8443
8444           if (clearing_check_p)
8445             gen_check = gen_ld_c_clr;
8446           else
8447             gen_check = gen_ld_c_nc;
8448         }
8449       else
8450         {
8451           gen_check = gen_chk_s;
8452         }
8453     }
8454   else
8455     gcc_unreachable ();
8456
8457   gcc_assert (mode_no >= 0);
8458   return gen_check[mode_no];
8459 }
8460
8461 /* Return nonzero, if INSN needs branchy recovery check.  */
8462 static bool
8463 ia64_needs_block_p (ds_t ts)
8464 {
8465   if (ts & BEGIN_DATA)
8466     return !mflag_sched_spec_ldc;
8467
8468   gcc_assert ((ts & BEGIN_CONTROL) != 0);
8469
8470   return !(mflag_sched_spec_control_ldc && mflag_sched_spec_ldc);
8471 }
8472
8473 /* Generate (or regenerate) a recovery check for INSN.  */
8474 static rtx
8475 ia64_gen_spec_check (rtx_insn *insn, rtx_insn *label, ds_t ds)
8476 {
8477   rtx op1, pat, check_pat;
8478   gen_func_t gen_check;
8479   int mode_no;
8480
8481   mode_no = get_mode_no_for_insn (insn);
8482   gcc_assert (mode_no >= 0);
8483
8484   if (label)
8485     op1 = label;
8486   else
8487     {
8488       gcc_assert (!ia64_needs_block_p (ds));
8489       op1 = copy_rtx (recog_data.operand[1]);
8490     }
8491
8492   gen_check = get_spec_check_gen_function (ds, mode_no, label == NULL_RTX,
8493                                            true);
8494
8495   check_pat = gen_check (copy_rtx (recog_data.operand[0]), op1);
8496
8497   pat = PATTERN (insn);
8498   if (GET_CODE (pat) == COND_EXEC)
8499     check_pat = gen_rtx_COND_EXEC (VOIDmode, copy_rtx (COND_EXEC_TEST (pat)),
8500                                    check_pat);
8501
8502   return check_pat;
8503 }
8504
8505 /* Return nonzero, if X is branchy recovery check.  */
8506 static int
8507 ia64_spec_check_p (rtx x)
8508 {
8509   x = PATTERN (x);
8510   if (GET_CODE (x) == COND_EXEC)
8511     x = COND_EXEC_CODE (x);
8512   if (GET_CODE (x) == SET)
8513     return ia64_spec_check_src_p (SET_SRC (x));
8514   return 0;
8515 }
8516
8517 /* Return nonzero, if SRC belongs to recovery check.  */
8518 static int
8519 ia64_spec_check_src_p (rtx src)
8520 {
8521   if (GET_CODE (src) == IF_THEN_ELSE)
8522     {
8523       rtx t;
8524
8525       t = XEXP (src, 0);
8526       if (GET_CODE (t) == NE)
8527         {
8528           t = XEXP (t, 0);
8529
8530           if (GET_CODE (t) == UNSPEC)
8531             {
8532               int code;
8533
8534               code = XINT (t, 1);
8535
8536               if (code == UNSPEC_LDCCLR
8537                   || code == UNSPEC_LDCNC
8538                   || code == UNSPEC_CHKACLR
8539                   || code == UNSPEC_CHKANC
8540                   || code == UNSPEC_CHKS)
8541                 {
8542                   gcc_assert (code != 0);
8543                   return code;
8544                 }
8545             }
8546         }
8547     }
8548   return 0;
8549 }
8550 \f
8551
8552 /* The following page contains abstract data `bundle states' which are
8553    used for bundling insns (inserting nops and template generation).  */
8554
8555 /* The following describes state of insn bundling.  */
8556
8557 struct bundle_state
8558 {
8559   /* Unique bundle state number to identify them in the debugging
8560      output  */
8561   int unique_num;
8562   rtx_insn *insn; /* corresponding insn, NULL for the 1st and the last state  */
8563   /* number nops before and after the insn  */
8564   short before_nops_num, after_nops_num;
8565   int insn_num; /* insn number (0 - for initial state, 1 - for the 1st
8566                    insn */
8567   int cost;     /* cost of the state in cycles */
8568   int accumulated_insns_num; /* number of all previous insns including
8569                                 nops.  L is considered as 2 insns */
8570   int branch_deviation; /* deviation of previous branches from 3rd slots  */
8571   int middle_bundle_stops; /* number of stop bits in the middle of bundles */
8572   struct bundle_state *next;  /* next state with the same insn_num  */
8573   struct bundle_state *originator; /* originator (previous insn state)  */
8574   /* All bundle states are in the following chain.  */
8575   struct bundle_state *allocated_states_chain;
8576   /* The DFA State after issuing the insn and the nops.  */
8577   state_t dfa_state;
8578 };
8579
8580 /* The following is map insn number to the corresponding bundle state.  */
8581
8582 static struct bundle_state **index_to_bundle_states;
8583
8584 /* The unique number of next bundle state.  */
8585
8586 static int bundle_states_num;
8587
8588 /* All allocated bundle states are in the following chain.  */
8589
8590 static struct bundle_state *allocated_bundle_states_chain;
8591
8592 /* All allocated but not used bundle states are in the following
8593    chain.  */
8594
8595 static struct bundle_state *free_bundle_state_chain;
8596
8597
8598 /* The following function returns a free bundle state.  */
8599
8600 static struct bundle_state *
8601 get_free_bundle_state (void)
8602 {
8603   struct bundle_state *result;
8604
8605   if (free_bundle_state_chain != NULL)
8606     {
8607       result = free_bundle_state_chain;
8608       free_bundle_state_chain = result->next;
8609     }
8610   else
8611     {
8612       result = XNEW (struct bundle_state);
8613       result->dfa_state = xmalloc (dfa_state_size);
8614       result->allocated_states_chain = allocated_bundle_states_chain;
8615       allocated_bundle_states_chain = result;
8616     }
8617   result->unique_num = bundle_states_num++;
8618   return result;
8619
8620 }
8621
8622 /* The following function frees given bundle state.  */
8623
8624 static void
8625 free_bundle_state (struct bundle_state *state)
8626 {
8627   state->next = free_bundle_state_chain;
8628   free_bundle_state_chain = state;
8629 }
8630
8631 /* Start work with abstract data `bundle states'.  */
8632
8633 static void
8634 initiate_bundle_states (void)
8635 {
8636   bundle_states_num = 0;
8637   free_bundle_state_chain = NULL;
8638   allocated_bundle_states_chain = NULL;
8639 }
8640
8641 /* Finish work with abstract data `bundle states'.  */
8642
8643 static void
8644 finish_bundle_states (void)
8645 {
8646   struct bundle_state *curr_state, *next_state;
8647
8648   for (curr_state = allocated_bundle_states_chain;
8649        curr_state != NULL;
8650        curr_state = next_state)
8651     {
8652       next_state = curr_state->allocated_states_chain;
8653       free (curr_state->dfa_state);
8654       free (curr_state);
8655     }
8656 }
8657
8658 /* Hashtable helpers.  */
8659
8660 struct bundle_state_hasher : nofree_ptr_hash <bundle_state>
8661 {
8662   static inline hashval_t hash (const bundle_state *);
8663   static inline bool equal (const bundle_state *, const bundle_state *);
8664 };
8665
8666 /* The function returns hash of BUNDLE_STATE.  */
8667
8668 inline hashval_t
8669 bundle_state_hasher::hash (const bundle_state *state)
8670 {
8671   unsigned result, i;
8672
8673   for (result = i = 0; i < dfa_state_size; i++)
8674     result += (((unsigned char *) state->dfa_state) [i]
8675                << ((i % CHAR_BIT) * 3 + CHAR_BIT));
8676   return result + state->insn_num;
8677 }
8678
8679 /* The function returns nonzero if the bundle state keys are equal.  */
8680
8681 inline bool
8682 bundle_state_hasher::equal (const bundle_state *state1,
8683                             const bundle_state *state2)
8684 {
8685   return (state1->insn_num == state2->insn_num
8686           && memcmp (state1->dfa_state, state2->dfa_state,
8687                      dfa_state_size) == 0);
8688 }
8689
8690 /* Hash table of the bundle states.  The key is dfa_state and insn_num
8691    of the bundle states.  */
8692
8693 static hash_table<bundle_state_hasher> *bundle_state_table;
8694
8695 /* The function inserts the BUNDLE_STATE into the hash table.  The
8696    function returns nonzero if the bundle has been inserted into the
8697    table.  The table contains the best bundle state with given key.  */
8698
8699 static int
8700 insert_bundle_state (struct bundle_state *bundle_state)
8701 {
8702   struct bundle_state **entry_ptr;
8703
8704   entry_ptr = bundle_state_table->find_slot (bundle_state, INSERT);
8705   if (*entry_ptr == NULL)
8706     {
8707       bundle_state->next = index_to_bundle_states [bundle_state->insn_num];
8708       index_to_bundle_states [bundle_state->insn_num] = bundle_state;
8709       *entry_ptr = bundle_state;
8710       return TRUE;
8711     }
8712   else if (bundle_state->cost < (*entry_ptr)->cost
8713            || (bundle_state->cost == (*entry_ptr)->cost
8714                && ((*entry_ptr)->accumulated_insns_num
8715                    > bundle_state->accumulated_insns_num
8716                    || ((*entry_ptr)->accumulated_insns_num
8717                        == bundle_state->accumulated_insns_num
8718                        && ((*entry_ptr)->branch_deviation
8719                            > bundle_state->branch_deviation
8720                            || ((*entry_ptr)->branch_deviation
8721                                == bundle_state->branch_deviation
8722                                && (*entry_ptr)->middle_bundle_stops
8723                                > bundle_state->middle_bundle_stops))))))
8724
8725     {
8726       struct bundle_state temp;
8727
8728       temp = **entry_ptr;
8729       **entry_ptr = *bundle_state;
8730       (*entry_ptr)->next = temp.next;
8731       *bundle_state = temp;
8732     }
8733   return FALSE;
8734 }
8735
8736 /* Start work with the hash table.  */
8737
8738 static void
8739 initiate_bundle_state_table (void)
8740 {
8741   bundle_state_table = new hash_table<bundle_state_hasher> (50);
8742 }
8743
8744 /* Finish work with the hash table.  */
8745
8746 static void
8747 finish_bundle_state_table (void)
8748 {
8749   delete bundle_state_table;
8750   bundle_state_table = NULL;
8751 }
8752
8753 \f
8754
8755 /* The following variable is a insn `nop' used to check bundle states
8756    with different number of inserted nops.  */
8757
8758 static rtx_insn *ia64_nop;
8759
8760 /* The following function tries to issue NOPS_NUM nops for the current
8761    state without advancing processor cycle.  If it failed, the
8762    function returns FALSE and frees the current state.  */
8763
8764 static int
8765 try_issue_nops (struct bundle_state *curr_state, int nops_num)
8766 {
8767   int i;
8768
8769   for (i = 0; i < nops_num; i++)
8770     if (state_transition (curr_state->dfa_state, ia64_nop) >= 0)
8771       {
8772         free_bundle_state (curr_state);
8773         return FALSE;
8774       }
8775   return TRUE;
8776 }
8777
8778 /* The following function tries to issue INSN for the current
8779    state without advancing processor cycle.  If it failed, the
8780    function returns FALSE and frees the current state.  */
8781
8782 static int
8783 try_issue_insn (struct bundle_state *curr_state, rtx insn)
8784 {
8785   if (insn && state_transition (curr_state->dfa_state, insn) >= 0)
8786     {
8787       free_bundle_state (curr_state);
8788       return FALSE;
8789     }
8790   return TRUE;
8791 }
8792
8793 /* The following function tries to issue BEFORE_NOPS_NUM nops and INSN
8794    starting with ORIGINATOR without advancing processor cycle.  If
8795    TRY_BUNDLE_END_P is TRUE, the function also/only (if
8796    ONLY_BUNDLE_END_P is TRUE) tries to issue nops to fill all bundle.
8797    If it was successful, the function creates new bundle state and
8798    insert into the hash table and into `index_to_bundle_states'.  */
8799
8800 static void
8801 issue_nops_and_insn (struct bundle_state *originator, int before_nops_num,
8802                      rtx_insn *insn, int try_bundle_end_p,
8803                      int only_bundle_end_p)
8804 {
8805   struct bundle_state *curr_state;
8806
8807   curr_state = get_free_bundle_state ();
8808   memcpy (curr_state->dfa_state, originator->dfa_state, dfa_state_size);
8809   curr_state->insn = insn;
8810   curr_state->insn_num = originator->insn_num + 1;
8811   curr_state->cost = originator->cost;
8812   curr_state->originator = originator;
8813   curr_state->before_nops_num = before_nops_num;
8814   curr_state->after_nops_num = 0;
8815   curr_state->accumulated_insns_num
8816     = originator->accumulated_insns_num + before_nops_num;
8817   curr_state->branch_deviation = originator->branch_deviation;
8818   curr_state->middle_bundle_stops = originator->middle_bundle_stops;
8819   gcc_assert (insn);
8820   if (INSN_CODE (insn) == CODE_FOR_insn_group_barrier)
8821     {
8822       gcc_assert (GET_MODE (insn) != TImode);
8823       if (!try_issue_nops (curr_state, before_nops_num))
8824         return;
8825       if (!try_issue_insn (curr_state, insn))
8826         return;
8827       memcpy (temp_dfa_state, curr_state->dfa_state, dfa_state_size);
8828       if (curr_state->accumulated_insns_num % 3 != 0)
8829         curr_state->middle_bundle_stops++;
8830       if (state_transition (temp_dfa_state, dfa_pre_cycle_insn) >= 0
8831           && curr_state->accumulated_insns_num % 3 != 0)
8832         {
8833           free_bundle_state (curr_state);
8834           return;
8835         }
8836     }
8837   else if (GET_MODE (insn) != TImode)
8838     {
8839       if (!try_issue_nops (curr_state, before_nops_num))
8840         return;
8841       if (!try_issue_insn (curr_state, insn))
8842         return;
8843       curr_state->accumulated_insns_num++;
8844       gcc_assert (!unknown_for_bundling_p (insn));
8845
8846       if (ia64_safe_type (insn) == TYPE_L)
8847         curr_state->accumulated_insns_num++;
8848     }
8849   else
8850     {
8851       /* If this is an insn that must be first in a group, then don't allow
8852          nops to be emitted before it.  Currently, alloc is the only such
8853          supported instruction.  */
8854       /* ??? The bundling automatons should handle this for us, but they do
8855          not yet have support for the first_insn attribute.  */
8856       if (before_nops_num > 0 && get_attr_first_insn (insn) == FIRST_INSN_YES)
8857         {
8858           free_bundle_state (curr_state);
8859           return;
8860         }
8861
8862       state_transition (curr_state->dfa_state, dfa_pre_cycle_insn);
8863       state_transition (curr_state->dfa_state, NULL);
8864       curr_state->cost++;
8865       if (!try_issue_nops (curr_state, before_nops_num))
8866         return;
8867       if (!try_issue_insn (curr_state, insn))
8868         return;
8869       curr_state->accumulated_insns_num++;
8870       if (unknown_for_bundling_p (insn))
8871         {
8872           /* Finish bundle containing asm insn.  */
8873           curr_state->after_nops_num
8874             = 3 - curr_state->accumulated_insns_num % 3;
8875           curr_state->accumulated_insns_num
8876             += 3 - curr_state->accumulated_insns_num % 3;
8877         }
8878       else if (ia64_safe_type (insn) == TYPE_L)
8879         curr_state->accumulated_insns_num++;
8880     }
8881   if (ia64_safe_type (insn) == TYPE_B)
8882     curr_state->branch_deviation
8883       += 2 - (curr_state->accumulated_insns_num - 1) % 3;
8884   if (try_bundle_end_p && curr_state->accumulated_insns_num % 3 != 0)
8885     {
8886       if (!only_bundle_end_p && insert_bundle_state (curr_state))
8887         {
8888           state_t dfa_state;
8889           struct bundle_state *curr_state1;
8890           struct bundle_state *allocated_states_chain;
8891
8892           curr_state1 = get_free_bundle_state ();
8893           dfa_state = curr_state1->dfa_state;
8894           allocated_states_chain = curr_state1->allocated_states_chain;
8895           *curr_state1 = *curr_state;
8896           curr_state1->dfa_state = dfa_state;
8897           curr_state1->allocated_states_chain = allocated_states_chain;
8898           memcpy (curr_state1->dfa_state, curr_state->dfa_state,
8899                   dfa_state_size);
8900           curr_state = curr_state1;
8901         }
8902       if (!try_issue_nops (curr_state,
8903                            3 - curr_state->accumulated_insns_num % 3))
8904         return;
8905       curr_state->after_nops_num
8906         = 3 - curr_state->accumulated_insns_num % 3;
8907       curr_state->accumulated_insns_num
8908         += 3 - curr_state->accumulated_insns_num % 3;
8909     }
8910   if (!insert_bundle_state (curr_state))
8911     free_bundle_state (curr_state);
8912   return;
8913 }
8914
8915 /* The following function returns position in the two window bundle
8916    for given STATE.  */
8917
8918 static int
8919 get_max_pos (state_t state)
8920 {
8921   if (cpu_unit_reservation_p (state, pos_6))
8922     return 6;
8923   else if (cpu_unit_reservation_p (state, pos_5))
8924     return 5;
8925   else if (cpu_unit_reservation_p (state, pos_4))
8926     return 4;
8927   else if (cpu_unit_reservation_p (state, pos_3))
8928     return 3;
8929   else if (cpu_unit_reservation_p (state, pos_2))
8930     return 2;
8931   else if (cpu_unit_reservation_p (state, pos_1))
8932     return 1;
8933   else
8934     return 0;
8935 }
8936
8937 /* The function returns code of a possible template for given position
8938    and state.  The function should be called only with 2 values of
8939    position equal to 3 or 6.  We avoid generating F NOPs by putting
8940    templates containing F insns at the end of the template search
8941    because undocumented anomaly in McKinley derived cores which can
8942    cause stalls if an F-unit insn (including a NOP) is issued within a
8943    six-cycle window after reading certain application registers (such
8944    as ar.bsp).  Furthermore, power-considerations also argue against
8945    the use of F-unit instructions unless they're really needed.  */
8946
8947 static int
8948 get_template (state_t state, int pos)
8949 {
8950   switch (pos)
8951     {
8952     case 3:
8953       if (cpu_unit_reservation_p (state, _0mmi_))
8954         return 1;
8955       else if (cpu_unit_reservation_p (state, _0mii_))
8956         return 0;
8957       else if (cpu_unit_reservation_p (state, _0mmb_))
8958         return 7;
8959       else if (cpu_unit_reservation_p (state, _0mib_))
8960         return 6;
8961       else if (cpu_unit_reservation_p (state, _0mbb_))
8962         return 5;
8963       else if (cpu_unit_reservation_p (state, _0bbb_))
8964         return 4;
8965       else if (cpu_unit_reservation_p (state, _0mmf_))
8966         return 3;
8967       else if (cpu_unit_reservation_p (state, _0mfi_))
8968         return 2;
8969       else if (cpu_unit_reservation_p (state, _0mfb_))
8970         return 8;
8971       else if (cpu_unit_reservation_p (state, _0mlx_))
8972         return 9;
8973       else
8974         gcc_unreachable ();
8975     case 6:
8976       if (cpu_unit_reservation_p (state, _1mmi_))
8977         return 1;
8978       else if (cpu_unit_reservation_p (state, _1mii_))
8979         return 0;
8980       else if (cpu_unit_reservation_p (state, _1mmb_))
8981         return 7;
8982       else if (cpu_unit_reservation_p (state, _1mib_))
8983         return 6;
8984       else if (cpu_unit_reservation_p (state, _1mbb_))
8985         return 5;
8986       else if (cpu_unit_reservation_p (state, _1bbb_))
8987         return 4;
8988       else if (_1mmf_ >= 0 && cpu_unit_reservation_p (state, _1mmf_))
8989         return 3;
8990       else if (cpu_unit_reservation_p (state, _1mfi_))
8991         return 2;
8992       else if (cpu_unit_reservation_p (state, _1mfb_))
8993         return 8;
8994       else if (cpu_unit_reservation_p (state, _1mlx_))
8995         return 9;
8996       else
8997         gcc_unreachable ();
8998     default:
8999       gcc_unreachable ();
9000     }
9001 }
9002
9003 /* True when INSN is important for bundling.  */
9004
9005 static bool
9006 important_for_bundling_p (rtx_insn *insn)
9007 {
9008   return (INSN_P (insn)
9009           && ia64_safe_itanium_class (insn) != ITANIUM_CLASS_IGNORE
9010           && GET_CODE (PATTERN (insn)) != USE
9011           && GET_CODE (PATTERN (insn)) != CLOBBER);
9012 }
9013
9014 /* The following function returns an insn important for insn bundling
9015    followed by INSN and before TAIL.  */
9016
9017 static rtx_insn *
9018 get_next_important_insn (rtx_insn *insn, rtx_insn *tail)
9019 {
9020   for (; insn && insn != tail; insn = NEXT_INSN (insn))
9021     if (important_for_bundling_p (insn))
9022       return insn;
9023   return NULL;
9024 }
9025
9026 /* True when INSN is unknown, but important, for bundling.  */
9027
9028 static bool
9029 unknown_for_bundling_p (rtx_insn *insn)
9030 {
9031   return (INSN_P (insn)
9032           && ia64_safe_itanium_class (insn) == ITANIUM_CLASS_UNKNOWN
9033           && GET_CODE (PATTERN (insn)) != USE
9034           && GET_CODE (PATTERN (insn)) != CLOBBER);
9035 }
9036
9037 /* Add a bundle selector TEMPLATE0 before INSN.  */
9038
9039 static void
9040 ia64_add_bundle_selector_before (int template0, rtx_insn *insn)
9041 {
9042   rtx b = gen_bundle_selector (GEN_INT (template0));
9043
9044   ia64_emit_insn_before (b, insn);
9045 #if NR_BUNDLES == 10
9046   if ((template0 == 4 || template0 == 5)
9047       && ia64_except_unwind_info (&global_options) == UI_TARGET)
9048     {
9049       int i;
9050       rtx note = NULL_RTX;
9051
9052       /* In .mbb and .bbb bundles, check if CALL_INSN isn't in the
9053          first or second slot.  If it is and has REG_EH_NOTE set, copy it
9054          to following nops, as br.call sets rp to the address of following
9055          bundle and therefore an EH region end must be on a bundle
9056          boundary.  */
9057       insn = PREV_INSN (insn);
9058       for (i = 0; i < 3; i++)
9059         {
9060           do
9061             insn = next_active_insn (insn);
9062           while (NONJUMP_INSN_P (insn)
9063                  && get_attr_empty (insn) == EMPTY_YES);
9064           if (CALL_P (insn))
9065             note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
9066           else if (note)
9067             {
9068               int code;
9069
9070               gcc_assert ((code = recog_memoized (insn)) == CODE_FOR_nop
9071                           || code == CODE_FOR_nop_b);
9072               if (find_reg_note (insn, REG_EH_REGION, NULL_RTX))
9073                 note = NULL_RTX;
9074               else
9075                 add_reg_note (insn, REG_EH_REGION, XEXP (note, 0));
9076             }
9077         }
9078     }
9079 #endif
9080 }
9081
9082 /* The following function does insn bundling.  Bundling means
9083    inserting templates and nop insns to fit insn groups into permitted
9084    templates.  Instruction scheduling uses NDFA (non-deterministic
9085    finite automata) encoding informations about the templates and the
9086    inserted nops.  Nondeterminism of the automata permits follows
9087    all possible insn sequences very fast.
9088
9089    Unfortunately it is not possible to get information about inserting
9090    nop insns and used templates from the automata states.  The
9091    automata only says that we can issue an insn possibly inserting
9092    some nops before it and using some template.  Therefore insn
9093    bundling in this function is implemented by using DFA
9094    (deterministic finite automata).  We follow all possible insn
9095    sequences by inserting 0-2 nops (that is what the NDFA describe for
9096    insn scheduling) before/after each insn being bundled.  We know the
9097    start of simulated processor cycle from insn scheduling (insn
9098    starting a new cycle has TImode).
9099
9100    Simple implementation of insn bundling would create enormous
9101    number of possible insn sequences satisfying information about new
9102    cycle ticks taken from the insn scheduling.  To make the algorithm
9103    practical we use dynamic programming.  Each decision (about
9104    inserting nops and implicitly about previous decisions) is described
9105    by structure bundle_state (see above).  If we generate the same
9106    bundle state (key is automaton state after issuing the insns and
9107    nops for it), we reuse already generated one.  As consequence we
9108    reject some decisions which cannot improve the solution and
9109    reduce memory for the algorithm.
9110
9111    When we reach the end of EBB (extended basic block), we choose the
9112    best sequence and then, moving back in EBB, insert templates for
9113    the best alternative.  The templates are taken from querying
9114    automaton state for each insn in chosen bundle states.
9115
9116    So the algorithm makes two (forward and backward) passes through
9117    EBB.  */
9118
9119 static void
9120 bundling (FILE *dump, int verbose, rtx_insn *prev_head_insn, rtx_insn *tail)
9121 {
9122   struct bundle_state *curr_state, *next_state, *best_state;
9123   rtx_insn *insn, *next_insn;
9124   int insn_num;
9125   int i, bundle_end_p, only_bundle_end_p, asm_p;
9126   int pos = 0, max_pos, template0, template1;
9127   rtx_insn *b;
9128   enum attr_type type;
9129
9130   insn_num = 0;
9131   /* Count insns in the EBB.  */
9132   for (insn = NEXT_INSN (prev_head_insn);
9133        insn && insn != tail;
9134        insn = NEXT_INSN (insn))
9135     if (INSN_P (insn))
9136       insn_num++;
9137   if (insn_num == 0)
9138     return;
9139   bundling_p = 1;
9140   dfa_clean_insn_cache ();
9141   initiate_bundle_state_table ();
9142   index_to_bundle_states = XNEWVEC (struct bundle_state *, insn_num + 2);
9143   /* First (forward) pass -- generation of bundle states.  */
9144   curr_state = get_free_bundle_state ();
9145   curr_state->insn = NULL;
9146   curr_state->before_nops_num = 0;
9147   curr_state->after_nops_num = 0;
9148   curr_state->insn_num = 0;
9149   curr_state->cost = 0;
9150   curr_state->accumulated_insns_num = 0;
9151   curr_state->branch_deviation = 0;
9152   curr_state->middle_bundle_stops = 0;
9153   curr_state->next = NULL;
9154   curr_state->originator = NULL;
9155   state_reset (curr_state->dfa_state);
9156   index_to_bundle_states [0] = curr_state;
9157   insn_num = 0;
9158   /* Shift cycle mark if it is put on insn which could be ignored.  */
9159   for (insn = NEXT_INSN (prev_head_insn);
9160        insn != tail;
9161        insn = NEXT_INSN (insn))
9162     if (INSN_P (insn)
9163         && !important_for_bundling_p (insn)
9164         && GET_MODE (insn) == TImode)
9165       {
9166         PUT_MODE (insn, VOIDmode);
9167         for (next_insn = NEXT_INSN (insn);
9168              next_insn != tail;
9169              next_insn = NEXT_INSN (next_insn))
9170           if (important_for_bundling_p (next_insn)
9171               && INSN_CODE (next_insn) != CODE_FOR_insn_group_barrier)
9172             {
9173               PUT_MODE (next_insn, TImode);
9174               break;
9175             }
9176       }
9177   /* Forward pass: generation of bundle states.  */
9178   for (insn = get_next_important_insn (NEXT_INSN (prev_head_insn), tail);
9179        insn != NULL_RTX;
9180        insn = next_insn)
9181     {
9182       gcc_assert (important_for_bundling_p (insn));
9183       type = ia64_safe_type (insn);
9184       next_insn = get_next_important_insn (NEXT_INSN (insn), tail);
9185       insn_num++;
9186       index_to_bundle_states [insn_num] = NULL;
9187       for (curr_state = index_to_bundle_states [insn_num - 1];
9188            curr_state != NULL;
9189            curr_state = next_state)
9190         {
9191           pos = curr_state->accumulated_insns_num % 3;
9192           next_state = curr_state->next;
9193           /* We must fill up the current bundle in order to start a
9194              subsequent asm insn in a new bundle.  Asm insn is always
9195              placed in a separate bundle.  */
9196           only_bundle_end_p
9197             = (next_insn != NULL_RTX
9198                && INSN_CODE (insn) == CODE_FOR_insn_group_barrier
9199                && unknown_for_bundling_p (next_insn));
9200           /* We may fill up the current bundle if it is the cycle end
9201              without a group barrier.  */
9202           bundle_end_p
9203             = (only_bundle_end_p || next_insn == NULL_RTX
9204                || (GET_MODE (next_insn) == TImode
9205                    && INSN_CODE (insn) != CODE_FOR_insn_group_barrier));
9206           if (type == TYPE_F || type == TYPE_B || type == TYPE_L
9207               || type == TYPE_S)
9208             issue_nops_and_insn (curr_state, 2, insn, bundle_end_p,
9209                                  only_bundle_end_p);
9210           issue_nops_and_insn (curr_state, 1, insn, bundle_end_p,
9211                                only_bundle_end_p);
9212           issue_nops_and_insn (curr_state, 0, insn, bundle_end_p,
9213                                only_bundle_end_p);
9214         }
9215       gcc_assert (index_to_bundle_states [insn_num]);
9216       for (curr_state = index_to_bundle_states [insn_num];
9217            curr_state != NULL;
9218            curr_state = curr_state->next)
9219         if (verbose >= 2 && dump)
9220           {
9221             /* This structure is taken from generated code of the
9222                pipeline hazard recognizer (see file insn-attrtab.c).
9223                Please don't forget to change the structure if a new
9224                automaton is added to .md file.  */
9225             struct DFA_chip
9226             {
9227               unsigned short one_automaton_state;
9228               unsigned short oneb_automaton_state;
9229               unsigned short two_automaton_state;
9230               unsigned short twob_automaton_state;
9231             };
9232
9233             fprintf
9234               (dump,
9235                "//    Bundle state %d (orig %d, cost %d, nops %d/%d, insns %d, branch %d, mid.stops %d state %d) for %d\n",
9236                curr_state->unique_num,
9237                (curr_state->originator == NULL
9238                 ? -1 : curr_state->originator->unique_num),
9239                curr_state->cost,
9240                curr_state->before_nops_num, curr_state->after_nops_num,
9241                curr_state->accumulated_insns_num, curr_state->branch_deviation,
9242                curr_state->middle_bundle_stops,
9243                ((struct DFA_chip *) curr_state->dfa_state)->twob_automaton_state,
9244                INSN_UID (insn));
9245           }
9246     }
9247
9248   /* We should find a solution because the 2nd insn scheduling has
9249      found one.  */
9250   gcc_assert (index_to_bundle_states [insn_num]);
9251   /* Find a state corresponding to the best insn sequence.  */
9252   best_state = NULL;
9253   for (curr_state = index_to_bundle_states [insn_num];
9254        curr_state != NULL;
9255        curr_state = curr_state->next)
9256     /* We are just looking at the states with fully filled up last
9257        bundle.  The first we prefer insn sequences with minimal cost
9258        then with minimal inserted nops and finally with branch insns
9259        placed in the 3rd slots.  */
9260     if (curr_state->accumulated_insns_num % 3 == 0
9261         && (best_state == NULL || best_state->cost > curr_state->cost
9262             || (best_state->cost == curr_state->cost
9263                 && (curr_state->accumulated_insns_num
9264                     < best_state->accumulated_insns_num
9265                     || (curr_state->accumulated_insns_num
9266                         == best_state->accumulated_insns_num
9267                         && (curr_state->branch_deviation
9268                             < best_state->branch_deviation
9269                             || (curr_state->branch_deviation
9270                                 == best_state->branch_deviation
9271                                 && curr_state->middle_bundle_stops
9272                                 < best_state->middle_bundle_stops)))))))
9273       best_state = curr_state;
9274   /* Second (backward) pass: adding nops and templates.  */
9275   gcc_assert (best_state);
9276   insn_num = best_state->before_nops_num;
9277   template0 = template1 = -1;
9278   for (curr_state = best_state;
9279        curr_state->originator != NULL;
9280        curr_state = curr_state->originator)
9281     {
9282       insn = curr_state->insn;
9283       asm_p = unknown_for_bundling_p (insn);
9284       insn_num++;
9285       if (verbose >= 2 && dump)
9286         {
9287           struct DFA_chip
9288           {
9289             unsigned short one_automaton_state;
9290             unsigned short oneb_automaton_state;
9291             unsigned short two_automaton_state;
9292             unsigned short twob_automaton_state;
9293           };
9294
9295           fprintf
9296             (dump,
9297              "//    Best %d (orig %d, cost %d, nops %d/%d, insns %d, branch %d, mid.stops %d, state %d) for %d\n",
9298              curr_state->unique_num,
9299              (curr_state->originator == NULL
9300               ? -1 : curr_state->originator->unique_num),
9301              curr_state->cost,
9302              curr_state->before_nops_num, curr_state->after_nops_num,
9303              curr_state->accumulated_insns_num, curr_state->branch_deviation,
9304              curr_state->middle_bundle_stops,
9305              ((struct DFA_chip *) curr_state->dfa_state)->twob_automaton_state,
9306              INSN_UID (insn));
9307         }
9308       /* Find the position in the current bundle window.  The window can
9309          contain at most two bundles.  Two bundle window means that
9310          the processor will make two bundle rotation.  */
9311       max_pos = get_max_pos (curr_state->dfa_state);
9312       if (max_pos == 6
9313           /* The following (negative template number) means that the
9314              processor did one bundle rotation.  */
9315           || (max_pos == 3 && template0 < 0))
9316         {
9317           /* We are at the end of the window -- find template(s) for
9318              its bundle(s).  */
9319           pos = max_pos;
9320           if (max_pos == 3)
9321             template0 = get_template (curr_state->dfa_state, 3);
9322           else
9323             {
9324               template1 = get_template (curr_state->dfa_state, 3);
9325               template0 = get_template (curr_state->dfa_state, 6);
9326             }
9327         }
9328       if (max_pos > 3 && template1 < 0)
9329         /* It may happen when we have the stop inside a bundle.  */
9330         {
9331           gcc_assert (pos <= 3);
9332           template1 = get_template (curr_state->dfa_state, 3);
9333           pos += 3;
9334         }
9335       if (!asm_p)
9336         /* Emit nops after the current insn.  */
9337         for (i = 0; i < curr_state->after_nops_num; i++)
9338           {
9339             rtx nop_pat = gen_nop ();
9340             rtx_insn *nop = emit_insn_after (nop_pat, insn);
9341             pos--;
9342             gcc_assert (pos >= 0);
9343             if (pos % 3 == 0)
9344               {
9345                 /* We are at the start of a bundle: emit the template
9346                    (it should be defined).  */
9347                 gcc_assert (template0 >= 0);
9348                 ia64_add_bundle_selector_before (template0, nop);
9349                 /* If we have two bundle window, we make one bundle
9350                    rotation.  Otherwise template0 will be undefined
9351                    (negative value).  */
9352                 template0 = template1;
9353                 template1 = -1;
9354               }
9355           }
9356       /* Move the position backward in the window.  Group barrier has
9357          no slot.  Asm insn takes all bundle.  */
9358       if (INSN_CODE (insn) != CODE_FOR_insn_group_barrier
9359           && !unknown_for_bundling_p (insn))
9360         pos--;
9361       /* Long insn takes 2 slots.  */
9362       if (ia64_safe_type (insn) == TYPE_L)
9363         pos--;
9364       gcc_assert (pos >= 0);
9365       if (pos % 3 == 0
9366           && INSN_CODE (insn) != CODE_FOR_insn_group_barrier
9367           && !unknown_for_bundling_p (insn))
9368         {
9369           /* The current insn is at the bundle start: emit the
9370              template.  */
9371           gcc_assert (template0 >= 0);
9372           ia64_add_bundle_selector_before (template0, insn);
9373           b = PREV_INSN (insn);
9374           insn = b;
9375           /* See comment above in analogous place for emitting nops
9376              after the insn.  */
9377           template0 = template1;
9378           template1 = -1;
9379         }
9380       /* Emit nops after the current insn.  */
9381       for (i = 0; i < curr_state->before_nops_num; i++)
9382         {
9383           rtx nop_pat = gen_nop ();
9384           ia64_emit_insn_before (nop_pat, insn);
9385           rtx_insn *nop = PREV_INSN (insn);
9386           insn = nop;
9387           pos--;
9388           gcc_assert (pos >= 0);
9389           if (pos % 3 == 0)
9390             {
9391               /* See comment above in analogous place for emitting nops
9392                  after the insn.  */
9393               gcc_assert (template0 >= 0);
9394               ia64_add_bundle_selector_before (template0, insn);
9395               b = PREV_INSN (insn);
9396               insn = b;
9397               template0 = template1;
9398               template1 = -1;
9399             }
9400         }
9401     }
9402
9403   if (flag_checking)
9404     {
9405       /* Assert right calculation of middle_bundle_stops.  */
9406       int num = best_state->middle_bundle_stops;
9407       bool start_bundle = true, end_bundle = false;
9408
9409       for (insn = NEXT_INSN (prev_head_insn);
9410            insn && insn != tail;
9411            insn = NEXT_INSN (insn))
9412         {
9413           if (!INSN_P (insn))
9414             continue;
9415           if (recog_memoized (insn) == CODE_FOR_bundle_selector)
9416             start_bundle = true;
9417           else
9418             {
9419               rtx_insn *next_insn;
9420
9421               for (next_insn = NEXT_INSN (insn);
9422                    next_insn && next_insn != tail;
9423                    next_insn = NEXT_INSN (next_insn))
9424                 if (INSN_P (next_insn)
9425                     && (ia64_safe_itanium_class (next_insn)
9426                         != ITANIUM_CLASS_IGNORE
9427                         || recog_memoized (next_insn)
9428                         == CODE_FOR_bundle_selector)
9429                     && GET_CODE (PATTERN (next_insn)) != USE
9430                     && GET_CODE (PATTERN (next_insn)) != CLOBBER)
9431                   break;
9432
9433               end_bundle = next_insn == NULL_RTX
9434                 || next_insn == tail
9435                 || (INSN_P (next_insn)
9436                     && recog_memoized (next_insn) == CODE_FOR_bundle_selector);
9437               if (recog_memoized (insn) == CODE_FOR_insn_group_barrier
9438                   && !start_bundle && !end_bundle
9439                   && next_insn
9440                   && !unknown_for_bundling_p (next_insn))
9441                 num--;
9442
9443               start_bundle = false;
9444             }
9445         }
9446
9447       gcc_assert (num == 0);
9448     }
9449
9450   free (index_to_bundle_states);
9451   finish_bundle_state_table ();
9452   bundling_p = 0;
9453   dfa_clean_insn_cache ();
9454 }
9455
9456 /* The following function is called at the end of scheduling BB or
9457    EBB.  After reload, it inserts stop bits and does insn bundling.  */
9458
9459 static void
9460 ia64_sched_finish (FILE *dump, int sched_verbose)
9461 {
9462   if (sched_verbose)
9463     fprintf (dump, "// Finishing schedule.\n");
9464   if (!reload_completed)
9465     return;
9466   if (reload_completed)
9467     {
9468       final_emit_insn_group_barriers (dump);
9469       bundling (dump, sched_verbose, current_sched_info->prev_head,
9470                 current_sched_info->next_tail);
9471       if (sched_verbose && dump)
9472         fprintf (dump, "//    finishing %d-%d\n",
9473                  INSN_UID (NEXT_INSN (current_sched_info->prev_head)),
9474                  INSN_UID (PREV_INSN (current_sched_info->next_tail)));
9475
9476       return;
9477     }
9478 }
9479
9480 /* The following function inserts stop bits in scheduled BB or EBB.  */
9481
9482 static void
9483 final_emit_insn_group_barriers (FILE *dump ATTRIBUTE_UNUSED)
9484 {
9485   rtx_insn *insn;
9486   int need_barrier_p = 0;
9487   int seen_good_insn = 0;
9488
9489   init_insn_group_barriers ();
9490
9491   for (insn = NEXT_INSN (current_sched_info->prev_head);
9492        insn != current_sched_info->next_tail;
9493        insn = NEXT_INSN (insn))
9494     {
9495       if (BARRIER_P (insn))
9496         {
9497           rtx_insn *last = prev_active_insn (insn);
9498
9499           if (! last)
9500             continue;
9501           if (JUMP_TABLE_DATA_P (last))
9502             last = prev_active_insn (last);
9503           if (recog_memoized (last) != CODE_FOR_insn_group_barrier)
9504             emit_insn_after (gen_insn_group_barrier (GEN_INT (3)), last);
9505
9506           init_insn_group_barriers ();
9507           seen_good_insn = 0;
9508           need_barrier_p = 0;
9509         }
9510       else if (NONDEBUG_INSN_P (insn))
9511         {
9512           if (recog_memoized (insn) == CODE_FOR_insn_group_barrier)
9513             {
9514               init_insn_group_barriers ();
9515               seen_good_insn = 0;
9516               need_barrier_p = 0;
9517             }
9518           else if (need_barrier_p || group_barrier_needed (insn)
9519                    || (mflag_sched_stop_bits_after_every_cycle
9520                        && GET_MODE (insn) == TImode
9521                        && seen_good_insn))
9522             {
9523               if (TARGET_EARLY_STOP_BITS)
9524                 {
9525                   rtx_insn *last;
9526
9527                   for (last = insn;
9528                        last != current_sched_info->prev_head;
9529                        last = PREV_INSN (last))
9530                     if (INSN_P (last) && GET_MODE (last) == TImode
9531                         && stops_p [INSN_UID (last)])
9532                       break;
9533                   if (last == current_sched_info->prev_head)
9534                     last = insn;
9535                   last = prev_active_insn (last);
9536                   if (last
9537                       && recog_memoized (last) != CODE_FOR_insn_group_barrier)
9538                     emit_insn_after (gen_insn_group_barrier (GEN_INT (3)),
9539                                      last);
9540                   init_insn_group_barriers ();
9541                   for (last = NEXT_INSN (last);
9542                        last != insn;
9543                        last = NEXT_INSN (last))
9544                     if (INSN_P (last))
9545                       {
9546                         group_barrier_needed (last);
9547                         if (recog_memoized (last) >= 0
9548                             && important_for_bundling_p (last))
9549                           seen_good_insn = 1;
9550                       }
9551                 }
9552               else
9553                 {
9554                   emit_insn_before (gen_insn_group_barrier (GEN_INT (3)),
9555                                     insn);
9556                   init_insn_group_barriers ();
9557                   seen_good_insn = 0;
9558                 }
9559               group_barrier_needed (insn);
9560               if (recog_memoized (insn) >= 0
9561                   && important_for_bundling_p (insn))
9562                 seen_good_insn = 1;
9563             }
9564           else if (recog_memoized (insn) >= 0
9565                    && important_for_bundling_p (insn))
9566             seen_good_insn = 1;
9567           need_barrier_p = (CALL_P (insn) || unknown_for_bundling_p (insn));
9568         }
9569     }
9570 }
9571
9572 \f
9573
9574 /* If the following function returns TRUE, we will use the DFA
9575    insn scheduler.  */
9576
9577 static int
9578 ia64_first_cycle_multipass_dfa_lookahead (void)
9579 {
9580   return (reload_completed ? 6 : 4);
9581 }
9582
9583 /* The following function initiates variable `dfa_pre_cycle_insn'.  */
9584
9585 static void
9586 ia64_init_dfa_pre_cycle_insn (void)
9587 {
9588   if (temp_dfa_state == NULL)
9589     {
9590       dfa_state_size = state_size ();
9591       temp_dfa_state = xmalloc (dfa_state_size);
9592       prev_cycle_state = xmalloc (dfa_state_size);
9593     }
9594   dfa_pre_cycle_insn = make_insn_raw (gen_pre_cycle ());
9595   SET_PREV_INSN (dfa_pre_cycle_insn) = SET_NEXT_INSN (dfa_pre_cycle_insn) = NULL_RTX;
9596   recog_memoized (dfa_pre_cycle_insn);
9597   dfa_stop_insn = make_insn_raw (gen_insn_group_barrier (GEN_INT (3)));
9598   SET_PREV_INSN (dfa_stop_insn) = SET_NEXT_INSN (dfa_stop_insn) = NULL_RTX;
9599   recog_memoized (dfa_stop_insn);
9600 }
9601
9602 /* The following function returns the pseudo insn DFA_PRE_CYCLE_INSN
9603    used by the DFA insn scheduler.  */
9604
9605 static rtx
9606 ia64_dfa_pre_cycle_insn (void)
9607 {
9608   return dfa_pre_cycle_insn;
9609 }
9610
9611 /* The following function returns TRUE if PRODUCER (of type ilog or
9612    ld) produces address for CONSUMER (of type st or stf). */
9613
9614 int
9615 ia64_st_address_bypass_p (rtx_insn *producer, rtx_insn *consumer)
9616 {
9617   rtx dest, reg, mem;
9618
9619   gcc_assert (producer && consumer);
9620   dest = ia64_single_set (producer);
9621   gcc_assert (dest);
9622   reg = SET_DEST (dest);
9623   gcc_assert (reg);
9624   if (GET_CODE (reg) == SUBREG)
9625     reg = SUBREG_REG (reg);
9626   gcc_assert (GET_CODE (reg) == REG);
9627
9628   dest = ia64_single_set (consumer);
9629   gcc_assert (dest);
9630   mem = SET_DEST (dest);
9631   gcc_assert (mem && GET_CODE (mem) == MEM);
9632   return reg_mentioned_p (reg, mem);
9633 }
9634
9635 /* The following function returns TRUE if PRODUCER (of type ilog or
9636    ld) produces address for CONSUMER (of type ld or fld). */
9637
9638 int
9639 ia64_ld_address_bypass_p (rtx_insn *producer, rtx_insn *consumer)
9640 {
9641   rtx dest, src, reg, mem;
9642
9643   gcc_assert (producer && consumer);
9644   dest = ia64_single_set (producer);
9645   gcc_assert (dest);
9646   reg = SET_DEST (dest);
9647   gcc_assert (reg);
9648   if (GET_CODE (reg) == SUBREG)
9649     reg = SUBREG_REG (reg);
9650   gcc_assert (GET_CODE (reg) == REG);
9651
9652   src = ia64_single_set (consumer);
9653   gcc_assert (src);
9654   mem = SET_SRC (src);
9655   gcc_assert (mem);
9656
9657   if (GET_CODE (mem) == UNSPEC && XVECLEN (mem, 0) > 0)
9658     mem = XVECEXP (mem, 0, 0);
9659   else if (GET_CODE (mem) == IF_THEN_ELSE)
9660     /* ??? Is this bypass necessary for ld.c?  */
9661     {
9662       gcc_assert (XINT (XEXP (XEXP (mem, 0), 0), 1) == UNSPEC_LDCCLR);
9663       mem = XEXP (mem, 1);
9664     }
9665
9666   while (GET_CODE (mem) == SUBREG || GET_CODE (mem) == ZERO_EXTEND)
9667     mem = XEXP (mem, 0);
9668
9669   if (GET_CODE (mem) == UNSPEC)
9670     {
9671       int c = XINT (mem, 1);
9672
9673       gcc_assert (c == UNSPEC_LDA || c == UNSPEC_LDS || c == UNSPEC_LDS_A
9674                   || c == UNSPEC_LDSA);
9675       mem = XVECEXP (mem, 0, 0);
9676     }
9677
9678   /* Note that LO_SUM is used for GOT loads.  */
9679   gcc_assert (GET_CODE (mem) == LO_SUM || GET_CODE (mem) == MEM);
9680
9681   return reg_mentioned_p (reg, mem);
9682 }
9683
9684 /* The following function returns TRUE if INSN produces address for a
9685    load/store insn.  We will place such insns into M slot because it
9686    decreases its latency time.  */
9687
9688 int
9689 ia64_produce_address_p (rtx insn)
9690 {
9691   return insn->call;
9692 }
9693
9694 \f
9695 /* Emit pseudo-ops for the assembler to describe predicate relations.
9696    At present this assumes that we only consider predicate pairs to
9697    be mutex, and that the assembler can deduce proper values from
9698    straight-line code.  */
9699
9700 static void
9701 emit_predicate_relation_info (void)
9702 {
9703   basic_block bb;
9704
9705   FOR_EACH_BB_REVERSE_FN (bb, cfun)
9706     {
9707       int r;
9708       rtx_insn *head = BB_HEAD (bb);
9709
9710       /* We only need such notes at code labels.  */
9711       if (! LABEL_P (head))
9712         continue;
9713       if (NOTE_INSN_BASIC_BLOCK_P (NEXT_INSN (head)))
9714         head = NEXT_INSN (head);
9715
9716       /* Skip p0, which may be thought to be live due to (reg:DI p0)
9717          grabbing the entire block of predicate registers.  */
9718       for (r = PR_REG (2); r < PR_REG (64); r += 2)
9719         if (REGNO_REG_SET_P (df_get_live_in (bb), r))
9720           {
9721             rtx p = gen_rtx_REG (BImode, r);
9722             rtx_insn *n = emit_insn_after (gen_pred_rel_mutex (p), head);
9723             if (head == BB_END (bb))
9724               BB_END (bb) = n;
9725             head = n;
9726           }
9727     }
9728
9729   /* Look for conditional calls that do not return, and protect predicate
9730      relations around them.  Otherwise the assembler will assume the call
9731      returns, and complain about uses of call-clobbered predicates after
9732      the call.  */
9733   FOR_EACH_BB_REVERSE_FN (bb, cfun)
9734     {
9735       rtx_insn *insn = BB_HEAD (bb);
9736
9737       while (1)
9738         {
9739           if (CALL_P (insn)
9740               && GET_CODE (PATTERN (insn)) == COND_EXEC
9741               && find_reg_note (insn, REG_NORETURN, NULL_RTX))
9742             {
9743               rtx_insn *b =
9744                 emit_insn_before (gen_safe_across_calls_all (), insn);
9745               rtx_insn *a = emit_insn_after (gen_safe_across_calls_normal (), insn);
9746               if (BB_HEAD (bb) == insn)
9747                 BB_HEAD (bb) = b;
9748               if (BB_END (bb) == insn)
9749                 BB_END (bb) = a;
9750             }
9751
9752           if (insn == BB_END (bb))
9753             break;
9754           insn = NEXT_INSN (insn);
9755         }
9756     }
9757 }
9758
9759 /* Perform machine dependent operations on the rtl chain INSNS.  */
9760
9761 static void
9762 ia64_reorg (void)
9763 {
9764   /* We are freeing block_for_insn in the toplev to keep compatibility
9765      with old MDEP_REORGS that are not CFG based.  Recompute it now.  */
9766   compute_bb_for_insn ();
9767
9768   /* If optimizing, we'll have split before scheduling.  */
9769   if (optimize == 0)
9770     split_all_insns ();
9771
9772   if (optimize && flag_schedule_insns_after_reload
9773       && dbg_cnt (ia64_sched2))
9774     {
9775       basic_block bb;
9776       timevar_push (TV_SCHED2);
9777       ia64_final_schedule = 1;
9778
9779       /* We can't let modulo-sched prevent us from scheduling any bbs,
9780          since we need the final schedule to produce bundle information.  */
9781       FOR_EACH_BB_FN (bb, cfun)
9782         bb->flags &= ~BB_DISABLE_SCHEDULE;
9783
9784       initiate_bundle_states ();
9785       ia64_nop = make_insn_raw (gen_nop ());
9786       SET_PREV_INSN (ia64_nop) = SET_NEXT_INSN (ia64_nop) = NULL_RTX;
9787       recog_memoized (ia64_nop);
9788       clocks_length = get_max_uid () + 1;
9789       stops_p = XCNEWVEC (char, clocks_length);
9790
9791       if (ia64_tune == PROCESSOR_ITANIUM2)
9792         {
9793           pos_1 = get_cpu_unit_code ("2_1");
9794           pos_2 = get_cpu_unit_code ("2_2");
9795           pos_3 = get_cpu_unit_code ("2_3");
9796           pos_4 = get_cpu_unit_code ("2_4");
9797           pos_5 = get_cpu_unit_code ("2_5");
9798           pos_6 = get_cpu_unit_code ("2_6");
9799           _0mii_ = get_cpu_unit_code ("2b_0mii.");
9800           _0mmi_ = get_cpu_unit_code ("2b_0mmi.");
9801           _0mfi_ = get_cpu_unit_code ("2b_0mfi.");
9802           _0mmf_ = get_cpu_unit_code ("2b_0mmf.");
9803           _0bbb_ = get_cpu_unit_code ("2b_0bbb.");
9804           _0mbb_ = get_cpu_unit_code ("2b_0mbb.");
9805           _0mib_ = get_cpu_unit_code ("2b_0mib.");
9806           _0mmb_ = get_cpu_unit_code ("2b_0mmb.");
9807           _0mfb_ = get_cpu_unit_code ("2b_0mfb.");
9808           _0mlx_ = get_cpu_unit_code ("2b_0mlx.");
9809           _1mii_ = get_cpu_unit_code ("2b_1mii.");
9810           _1mmi_ = get_cpu_unit_code ("2b_1mmi.");
9811           _1mfi_ = get_cpu_unit_code ("2b_1mfi.");
9812           _1mmf_ = get_cpu_unit_code ("2b_1mmf.");
9813           _1bbb_ = get_cpu_unit_code ("2b_1bbb.");
9814           _1mbb_ = get_cpu_unit_code ("2b_1mbb.");
9815           _1mib_ = get_cpu_unit_code ("2b_1mib.");
9816           _1mmb_ = get_cpu_unit_code ("2b_1mmb.");
9817           _1mfb_ = get_cpu_unit_code ("2b_1mfb.");
9818           _1mlx_ = get_cpu_unit_code ("2b_1mlx.");
9819         }
9820       else
9821         {
9822           pos_1 = get_cpu_unit_code ("1_1");
9823           pos_2 = get_cpu_unit_code ("1_2");
9824           pos_3 = get_cpu_unit_code ("1_3");
9825           pos_4 = get_cpu_unit_code ("1_4");
9826           pos_5 = get_cpu_unit_code ("1_5");
9827           pos_6 = get_cpu_unit_code ("1_6");
9828           _0mii_ = get_cpu_unit_code ("1b_0mii.");
9829           _0mmi_ = get_cpu_unit_code ("1b_0mmi.");
9830           _0mfi_ = get_cpu_unit_code ("1b_0mfi.");
9831           _0mmf_ = get_cpu_unit_code ("1b_0mmf.");
9832           _0bbb_ = get_cpu_unit_code ("1b_0bbb.");
9833           _0mbb_ = get_cpu_unit_code ("1b_0mbb.");
9834           _0mib_ = get_cpu_unit_code ("1b_0mib.");
9835           _0mmb_ = get_cpu_unit_code ("1b_0mmb.");
9836           _0mfb_ = get_cpu_unit_code ("1b_0mfb.");
9837           _0mlx_ = get_cpu_unit_code ("1b_0mlx.");
9838           _1mii_ = get_cpu_unit_code ("1b_1mii.");
9839           _1mmi_ = get_cpu_unit_code ("1b_1mmi.");
9840           _1mfi_ = get_cpu_unit_code ("1b_1mfi.");
9841           _1mmf_ = get_cpu_unit_code ("1b_1mmf.");
9842           _1bbb_ = get_cpu_unit_code ("1b_1bbb.");
9843           _1mbb_ = get_cpu_unit_code ("1b_1mbb.");
9844           _1mib_ = get_cpu_unit_code ("1b_1mib.");
9845           _1mmb_ = get_cpu_unit_code ("1b_1mmb.");
9846           _1mfb_ = get_cpu_unit_code ("1b_1mfb.");
9847           _1mlx_ = get_cpu_unit_code ("1b_1mlx.");
9848         }
9849
9850       if (flag_selective_scheduling2
9851           && !maybe_skip_selective_scheduling ())
9852         run_selective_scheduling ();
9853       else
9854         schedule_ebbs ();
9855
9856       /* Redo alignment computation, as it might gone wrong.  */
9857       compute_alignments ();
9858
9859       /* We cannot reuse this one because it has been corrupted by the
9860          evil glat.  */
9861       finish_bundle_states ();
9862       free (stops_p);
9863       stops_p = NULL;
9864       emit_insn_group_barriers (dump_file);
9865
9866       ia64_final_schedule = 0;
9867       timevar_pop (TV_SCHED2);
9868     }
9869   else
9870     emit_all_insn_group_barriers (dump_file);
9871
9872   df_analyze ();
9873
9874   /* A call must not be the last instruction in a function, so that the
9875      return address is still within the function, so that unwinding works
9876      properly.  Note that IA-64 differs from dwarf2 on this point.  */
9877   if (ia64_except_unwind_info (&global_options) == UI_TARGET)
9878     {
9879       rtx_insn *insn;
9880       int saw_stop = 0;
9881
9882       insn = get_last_insn ();
9883       if (! INSN_P (insn))
9884         insn = prev_active_insn (insn);
9885       if (insn)
9886         {
9887           /* Skip over insns that expand to nothing.  */
9888           while (NONJUMP_INSN_P (insn)
9889                  && get_attr_empty (insn) == EMPTY_YES)
9890             {
9891               if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
9892                   && XINT (PATTERN (insn), 1) == UNSPECV_INSN_GROUP_BARRIER)
9893                 saw_stop = 1;
9894               insn = prev_active_insn (insn);
9895             }
9896           if (CALL_P (insn))
9897             {
9898               if (! saw_stop)
9899                 emit_insn (gen_insn_group_barrier (GEN_INT (3)));
9900               emit_insn (gen_break_f ());
9901               emit_insn (gen_insn_group_barrier (GEN_INT (3)));
9902             }
9903         }
9904     }
9905
9906   emit_predicate_relation_info ();
9907
9908   if (flag_var_tracking)
9909     {
9910       timevar_push (TV_VAR_TRACKING);
9911       variable_tracking_main ();
9912       timevar_pop (TV_VAR_TRACKING);
9913     }
9914   df_finish_pass (false);
9915 }
9916 \f
9917 /* Return true if REGNO is used by the epilogue.  */
9918
9919 int
9920 ia64_epilogue_uses (int regno)
9921 {
9922   switch (regno)
9923     {
9924     case R_GR (1):
9925       /* With a call to a function in another module, we will write a new
9926          value to "gp".  After returning from such a call, we need to make
9927          sure the function restores the original gp-value, even if the
9928          function itself does not use the gp anymore.  */
9929       return !(TARGET_AUTO_PIC || TARGET_NO_PIC);
9930
9931     case IN_REG (0): case IN_REG (1): case IN_REG (2): case IN_REG (3):
9932     case IN_REG (4): case IN_REG (5): case IN_REG (6): case IN_REG (7):
9933       /* For functions defined with the syscall_linkage attribute, all
9934          input registers are marked as live at all function exits.  This
9935          prevents the register allocator from using the input registers,
9936          which in turn makes it possible to restart a system call after
9937          an interrupt without having to save/restore the input registers.
9938          This also prevents kernel data from leaking to application code.  */
9939       return lookup_attribute ("syscall_linkage",
9940            TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))) != NULL;
9941
9942     case R_BR (0):
9943       /* Conditional return patterns can't represent the use of `b0' as
9944          the return address, so we force the value live this way.  */
9945       return 1;
9946
9947     case AR_PFS_REGNUM:
9948       /* Likewise for ar.pfs, which is used by br.ret.  */
9949       return 1;
9950
9951     default:
9952       return 0;
9953     }
9954 }
9955
9956 /* Return true if REGNO is used by the frame unwinder.  */
9957
9958 int
9959 ia64_eh_uses (int regno)
9960 {
9961   unsigned int r;
9962
9963   if (! reload_completed)
9964     return 0;
9965
9966   if (regno == 0)
9967     return 0;
9968
9969   for (r = reg_save_b0; r <= reg_save_ar_lc; r++)
9970     if (regno == current_frame_info.r[r]
9971        || regno == emitted_frame_related_regs[r])
9972       return 1;
9973
9974   return 0;
9975 }
9976 \f
9977 /* Return true if this goes in small data/bss.  */
9978
9979 /* ??? We could also support own long data here.  Generating movl/add/ld8
9980    instead of addl,ld8/ld8.  This makes the code bigger, but should make the
9981    code faster because there is one less load.  This also includes incomplete
9982    types which can't go in sdata/sbss.  */
9983
9984 static bool
9985 ia64_in_small_data_p (const_tree exp)
9986 {
9987   if (TARGET_NO_SDATA)
9988     return false;
9989
9990   /* We want to merge strings, so we never consider them small data.  */
9991   if (TREE_CODE (exp) == STRING_CST)
9992     return false;
9993
9994   /* Functions are never small data.  */
9995   if (TREE_CODE (exp) == FUNCTION_DECL)
9996     return false;
9997
9998   if (TREE_CODE (exp) == VAR_DECL && DECL_SECTION_NAME (exp))
9999     {
10000       const char *section = DECL_SECTION_NAME (exp);
10001
10002       if (strcmp (section, ".sdata") == 0
10003           || strncmp (section, ".sdata.", 7) == 0
10004           || strncmp (section, ".gnu.linkonce.s.", 16) == 0
10005           || strcmp (section, ".sbss") == 0
10006           || strncmp (section, ".sbss.", 6) == 0
10007           || strncmp (section, ".gnu.linkonce.sb.", 17) == 0)
10008         return true;
10009     }
10010   else
10011     {
10012       HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
10013
10014       /* If this is an incomplete type with size 0, then we can't put it
10015          in sdata because it might be too big when completed.  */
10016       if (size > 0 && size <= ia64_section_threshold)
10017         return true;
10018     }
10019
10020   return false;
10021 }
10022 \f
10023 /* Output assembly directives for prologue regions.  */
10024
10025 /* The current basic block number.  */
10026
10027 static bool last_block;
10028
10029 /* True if we need a copy_state command at the start of the next block.  */
10030
10031 static bool need_copy_state;
10032
10033 #ifndef MAX_ARTIFICIAL_LABEL_BYTES
10034 # define MAX_ARTIFICIAL_LABEL_BYTES 30
10035 #endif
10036
10037 /* The function emits unwind directives for the start of an epilogue.  */
10038
10039 static void
10040 process_epilogue (FILE *asm_out_file, rtx insn ATTRIBUTE_UNUSED,
10041                   bool unwind, bool frame ATTRIBUTE_UNUSED)
10042 {
10043   /* If this isn't the last block of the function, then we need to label the
10044      current state, and copy it back in at the start of the next block.  */
10045
10046   if (!last_block)
10047     {
10048       if (unwind)
10049         fprintf (asm_out_file, "\t.label_state %d\n",
10050                  ++cfun->machine->state_num);
10051       need_copy_state = true;
10052     }
10053
10054   if (unwind)
10055     fprintf (asm_out_file, "\t.restore sp\n");
10056 }
10057
10058 /* This function processes a SET pattern for REG_CFA_ADJUST_CFA.  */
10059
10060 static void
10061 process_cfa_adjust_cfa (FILE *asm_out_file, rtx pat, rtx insn,
10062                         bool unwind, bool frame)
10063 {
10064   rtx dest = SET_DEST (pat);
10065   rtx src = SET_SRC (pat);
10066
10067   if (dest == stack_pointer_rtx)
10068     {
10069       if (GET_CODE (src) == PLUS)
10070         {
10071           rtx op0 = XEXP (src, 0);
10072           rtx op1 = XEXP (src, 1);
10073
10074           gcc_assert (op0 == dest && GET_CODE (op1) == CONST_INT);
10075
10076           if (INTVAL (op1) < 0)
10077             {
10078               gcc_assert (!frame_pointer_needed);
10079               if (unwind)
10080                 fprintf (asm_out_file,
10081                          "\t.fframe " HOST_WIDE_INT_PRINT_DEC"\n",
10082                          -INTVAL (op1));
10083             }
10084           else
10085             process_epilogue (asm_out_file, insn, unwind, frame);
10086         }
10087       else
10088         {
10089           gcc_assert (src == hard_frame_pointer_rtx);
10090           process_epilogue (asm_out_file, insn, unwind, frame);
10091         }
10092     }
10093   else if (dest == hard_frame_pointer_rtx)
10094     {
10095       gcc_assert (src == stack_pointer_rtx);
10096       gcc_assert (frame_pointer_needed);
10097
10098       if (unwind)
10099         fprintf (asm_out_file, "\t.vframe r%d\n",
10100                  ia64_dbx_register_number (REGNO (dest)));
10101     }
10102   else
10103     gcc_unreachable ();
10104 }
10105
10106 /* This function processes a SET pattern for REG_CFA_REGISTER.  */
10107
10108 static void
10109 process_cfa_register (FILE *asm_out_file, rtx pat, bool unwind)
10110 {
10111   rtx dest = SET_DEST (pat);
10112   rtx src = SET_SRC (pat);
10113   int dest_regno = REGNO (dest);
10114   int src_regno;
10115
10116   if (src == pc_rtx)
10117     {
10118       /* Saving return address pointer.  */
10119       if (unwind)
10120         fprintf (asm_out_file, "\t.save rp, r%d\n",
10121                  ia64_dbx_register_number (dest_regno));
10122       return;
10123     }
10124
10125   src_regno = REGNO (src);
10126
10127   switch (src_regno)
10128     {
10129     case PR_REG (0):
10130       gcc_assert (dest_regno == current_frame_info.r[reg_save_pr]);
10131       if (unwind)
10132         fprintf (asm_out_file, "\t.save pr, r%d\n",
10133                  ia64_dbx_register_number (dest_regno));
10134       break;
10135
10136     case AR_UNAT_REGNUM:
10137       gcc_assert (dest_regno == current_frame_info.r[reg_save_ar_unat]);
10138       if (unwind)
10139         fprintf (asm_out_file, "\t.save ar.unat, r%d\n",
10140                  ia64_dbx_register_number (dest_regno));
10141       break;
10142
10143     case AR_LC_REGNUM:
10144       gcc_assert (dest_regno == current_frame_info.r[reg_save_ar_lc]);
10145       if (unwind)
10146         fprintf (asm_out_file, "\t.save ar.lc, r%d\n",
10147                  ia64_dbx_register_number (dest_regno));
10148       break;
10149
10150     default:
10151       /* Everything else should indicate being stored to memory.  */
10152       gcc_unreachable ();
10153     }
10154 }
10155
10156 /* This function processes a SET pattern for REG_CFA_OFFSET.  */
10157
10158 static void
10159 process_cfa_offset (FILE *asm_out_file, rtx pat, bool unwind)
10160 {
10161   rtx dest = SET_DEST (pat);
10162   rtx src = SET_SRC (pat);
10163   int src_regno = REGNO (src);
10164   const char *saveop;
10165   HOST_WIDE_INT off;
10166   rtx base;
10167
10168   gcc_assert (MEM_P (dest));
10169   if (GET_CODE (XEXP (dest, 0)) == REG)
10170     {
10171       base = XEXP (dest, 0);
10172       off = 0;
10173     }
10174   else
10175     {
10176       gcc_assert (GET_CODE (XEXP (dest, 0)) == PLUS
10177                   && GET_CODE (XEXP (XEXP (dest, 0), 1)) == CONST_INT);
10178       base = XEXP (XEXP (dest, 0), 0);
10179       off = INTVAL (XEXP (XEXP (dest, 0), 1));
10180     }
10181
10182   if (base == hard_frame_pointer_rtx)
10183     {
10184       saveop = ".savepsp";
10185       off = - off;
10186     }
10187   else
10188     {
10189       gcc_assert (base == stack_pointer_rtx);
10190       saveop = ".savesp";
10191     }
10192
10193   src_regno = REGNO (src);
10194   switch (src_regno)
10195     {
10196     case BR_REG (0):
10197       gcc_assert (!current_frame_info.r[reg_save_b0]);
10198       if (unwind)
10199         fprintf (asm_out_file, "\t%s rp, " HOST_WIDE_INT_PRINT_DEC "\n",
10200                  saveop, off);
10201       break;
10202
10203     case PR_REG (0):
10204       gcc_assert (!current_frame_info.r[reg_save_pr]);
10205       if (unwind)
10206         fprintf (asm_out_file, "\t%s pr, " HOST_WIDE_INT_PRINT_DEC "\n",
10207                  saveop, off);
10208       break;
10209
10210     case AR_LC_REGNUM:
10211       gcc_assert (!current_frame_info.r[reg_save_ar_lc]);
10212       if (unwind)
10213         fprintf (asm_out_file, "\t%s ar.lc, " HOST_WIDE_INT_PRINT_DEC "\n",
10214                  saveop, off);
10215       break;
10216
10217     case AR_PFS_REGNUM:
10218       gcc_assert (!current_frame_info.r[reg_save_ar_pfs]);
10219       if (unwind)
10220         fprintf (asm_out_file, "\t%s ar.pfs, " HOST_WIDE_INT_PRINT_DEC "\n",
10221                  saveop, off);
10222       break;
10223
10224     case AR_UNAT_REGNUM:
10225       gcc_assert (!current_frame_info.r[reg_save_ar_unat]);
10226       if (unwind)
10227         fprintf (asm_out_file, "\t%s ar.unat, " HOST_WIDE_INT_PRINT_DEC "\n",
10228                  saveop, off);
10229       break;
10230
10231     case GR_REG (4):
10232     case GR_REG (5):
10233     case GR_REG (6):
10234     case GR_REG (7):
10235       if (unwind)
10236         fprintf (asm_out_file, "\t.save.g 0x%x\n",
10237                  1 << (src_regno - GR_REG (4)));
10238       break;
10239
10240     case BR_REG (1):
10241     case BR_REG (2):
10242     case BR_REG (3):
10243     case BR_REG (4):
10244     case BR_REG (5):
10245       if (unwind)
10246         fprintf (asm_out_file, "\t.save.b 0x%x\n",
10247                  1 << (src_regno - BR_REG (1)));
10248       break;
10249
10250     case FR_REG (2):
10251     case FR_REG (3):
10252     case FR_REG (4):
10253     case FR_REG (5):
10254       if (unwind)
10255         fprintf (asm_out_file, "\t.save.f 0x%x\n",
10256                  1 << (src_regno - FR_REG (2)));
10257       break;
10258
10259     case FR_REG (16): case FR_REG (17): case FR_REG (18): case FR_REG (19):
10260     case FR_REG (20): case FR_REG (21): case FR_REG (22): case FR_REG (23):
10261     case FR_REG (24): case FR_REG (25): case FR_REG (26): case FR_REG (27):
10262     case FR_REG (28): case FR_REG (29): case FR_REG (30): case FR_REG (31):
10263       if (unwind)
10264         fprintf (asm_out_file, "\t.save.gf 0x0, 0x%x\n",
10265                  1 << (src_regno - FR_REG (12)));
10266       break;
10267
10268     default:
10269       /* ??? For some reason we mark other general registers, even those
10270          we can't represent in the unwind info.  Ignore them.  */
10271       break;
10272     }
10273 }
10274
10275 /* This function looks at a single insn and emits any directives
10276    required to unwind this insn.  */
10277
10278 static void
10279 ia64_asm_unwind_emit (FILE *asm_out_file, rtx_insn *insn)
10280 {
10281   bool unwind = ia64_except_unwind_info (&global_options) == UI_TARGET;
10282   bool frame = dwarf2out_do_frame ();
10283   rtx note, pat;
10284   bool handled_one;
10285
10286   if (!unwind && !frame)
10287     return;
10288
10289   if (NOTE_INSN_BASIC_BLOCK_P (insn))
10290     {
10291       last_block = NOTE_BASIC_BLOCK (insn)->next_bb
10292      == EXIT_BLOCK_PTR_FOR_FN (cfun);
10293
10294       /* Restore unwind state from immediately before the epilogue.  */
10295       if (need_copy_state)
10296         {
10297           if (unwind)
10298             {
10299               fprintf (asm_out_file, "\t.body\n");
10300               fprintf (asm_out_file, "\t.copy_state %d\n",
10301                        cfun->machine->state_num);
10302             }
10303           need_copy_state = false;
10304         }
10305     }
10306
10307   if (NOTE_P (insn) || ! RTX_FRAME_RELATED_P (insn))
10308     return;
10309
10310   /* Look for the ALLOC insn.  */
10311   if (INSN_CODE (insn) == CODE_FOR_alloc)
10312     {
10313       rtx dest = SET_DEST (XVECEXP (PATTERN (insn), 0, 0));
10314       int dest_regno = REGNO (dest);
10315
10316       /* If this is the final destination for ar.pfs, then this must
10317          be the alloc in the prologue.  */
10318       if (dest_regno == current_frame_info.r[reg_save_ar_pfs])
10319         {
10320           if (unwind)
10321             fprintf (asm_out_file, "\t.save ar.pfs, r%d\n",
10322                      ia64_dbx_register_number (dest_regno));
10323         }
10324       else
10325         {
10326           /* This must be an alloc before a sibcall.  We must drop the
10327              old frame info.  The easiest way to drop the old frame
10328              info is to ensure we had a ".restore sp" directive
10329              followed by a new prologue.  If the procedure doesn't
10330              have a memory-stack frame, we'll issue a dummy ".restore
10331              sp" now.  */
10332           if (current_frame_info.total_size == 0 && !frame_pointer_needed)
10333             /* if haven't done process_epilogue() yet, do it now */
10334             process_epilogue (asm_out_file, insn, unwind, frame);
10335           if (unwind)
10336             fprintf (asm_out_file, "\t.prologue\n");
10337         }
10338       return;
10339     }
10340
10341   handled_one = false;
10342   for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
10343     switch (REG_NOTE_KIND (note))
10344       {
10345       case REG_CFA_ADJUST_CFA:
10346         pat = XEXP (note, 0);
10347         if (pat == NULL)
10348           pat = PATTERN (insn);
10349         process_cfa_adjust_cfa (asm_out_file, pat, insn, unwind, frame);
10350         handled_one = true;
10351         break;
10352
10353       case REG_CFA_OFFSET:
10354         pat = XEXP (note, 0);
10355         if (pat == NULL)
10356           pat = PATTERN (insn);
10357         process_cfa_offset (asm_out_file, pat, unwind);
10358         handled_one = true;
10359         break;
10360
10361       case REG_CFA_REGISTER:
10362         pat = XEXP (note, 0);
10363         if (pat == NULL)
10364           pat = PATTERN (insn);
10365         process_cfa_register (asm_out_file, pat, unwind);
10366         handled_one = true;
10367         break;
10368
10369       case REG_FRAME_RELATED_EXPR:
10370       case REG_CFA_DEF_CFA:
10371       case REG_CFA_EXPRESSION:
10372       case REG_CFA_RESTORE:
10373       case REG_CFA_SET_VDRAP:
10374         /* Not used in the ia64 port.  */
10375         gcc_unreachable ();
10376
10377       default:
10378         /* Not a frame-related note.  */
10379         break;
10380       }
10381
10382   /* All REG_FRAME_RELATED_P insns, besides ALLOC, are marked with the
10383      explicit action to take.  No guessing required.  */
10384   gcc_assert (handled_one);
10385 }
10386
10387 /* Implement TARGET_ASM_EMIT_EXCEPT_PERSONALITY.  */
10388
10389 static void
10390 ia64_asm_emit_except_personality (rtx personality)
10391 {
10392   fputs ("\t.personality\t", asm_out_file);
10393   output_addr_const (asm_out_file, personality);
10394   fputc ('\n', asm_out_file);
10395 }
10396
10397 /* Implement TARGET_ASM_INITIALIZE_SECTIONS.  */
10398
10399 static void
10400 ia64_asm_init_sections (void)
10401 {
10402   exception_section = get_unnamed_section (0, output_section_asm_op,
10403                                            "\t.handlerdata");
10404 }
10405
10406 /* Implement TARGET_DEBUG_UNWIND_INFO.  */
10407
10408 static enum unwind_info_type
10409 ia64_debug_unwind_info (void)
10410 {
10411   return UI_TARGET;
10412 }
10413 \f
10414 enum ia64_builtins
10415 {
10416   IA64_BUILTIN_BSP,
10417   IA64_BUILTIN_COPYSIGNQ,
10418   IA64_BUILTIN_FABSQ,
10419   IA64_BUILTIN_FLUSHRS,
10420   IA64_BUILTIN_INFQ,
10421   IA64_BUILTIN_HUGE_VALQ,
10422   IA64_BUILTIN_NANQ,
10423   IA64_BUILTIN_NANSQ,
10424   IA64_BUILTIN_max
10425 };
10426
10427 static GTY(()) tree ia64_builtins[(int) IA64_BUILTIN_max];
10428
10429 void
10430 ia64_init_builtins (void)
10431 {
10432   tree fpreg_type;
10433   tree float80_type;
10434   tree decl;
10435
10436   /* The __fpreg type.  */
10437   fpreg_type = make_node (REAL_TYPE);
10438   TYPE_PRECISION (fpreg_type) = 82;
10439   layout_type (fpreg_type);
10440   (*lang_hooks.types.register_builtin_type) (fpreg_type, "__fpreg");
10441
10442   /* The __float80 type.  */
10443   if (float64x_type_node != NULL_TREE
10444       && TYPE_MODE (float64x_type_node) == XFmode)
10445     float80_type = float64x_type_node;
10446   else
10447     {
10448       float80_type = make_node (REAL_TYPE);
10449       TYPE_PRECISION (float80_type) = 80;
10450       layout_type (float80_type);
10451     }
10452   (*lang_hooks.types.register_builtin_type) (float80_type, "__float80");
10453
10454   /* The __float128 type.  */
10455   if (!TARGET_HPUX)
10456     {
10457       tree ftype;
10458       tree const_string_type
10459         = build_pointer_type (build_qualified_type
10460                               (char_type_node, TYPE_QUAL_CONST));
10461
10462       (*lang_hooks.types.register_builtin_type) (float128_type_node,
10463                                                  "__float128");
10464
10465       /* TFmode support builtins.  */
10466       ftype = build_function_type_list (float128_type_node, NULL_TREE);
10467       decl = add_builtin_function ("__builtin_infq", ftype,
10468                                    IA64_BUILTIN_INFQ, BUILT_IN_MD,
10469                                    NULL, NULL_TREE);
10470       ia64_builtins[IA64_BUILTIN_INFQ] = decl;
10471
10472       decl = add_builtin_function ("__builtin_huge_valq", ftype,
10473                                    IA64_BUILTIN_HUGE_VALQ, BUILT_IN_MD,
10474                                    NULL, NULL_TREE);
10475       ia64_builtins[IA64_BUILTIN_HUGE_VALQ] = decl;
10476
10477       ftype = build_function_type_list (float128_type_node,
10478                                         const_string_type,
10479                                         NULL_TREE);
10480       decl = add_builtin_function ("__builtin_nanq", ftype,
10481                                    IA64_BUILTIN_NANQ, BUILT_IN_MD,
10482                                    "nanq", NULL_TREE);
10483       TREE_READONLY (decl) = 1;
10484       ia64_builtins[IA64_BUILTIN_NANQ] = decl;
10485
10486       decl = add_builtin_function ("__builtin_nansq", ftype,
10487                                    IA64_BUILTIN_NANSQ, BUILT_IN_MD,
10488                                    "nansq", NULL_TREE);
10489       TREE_READONLY (decl) = 1;
10490       ia64_builtins[IA64_BUILTIN_NANSQ] = decl;
10491
10492       ftype = build_function_type_list (float128_type_node,
10493                                         float128_type_node,
10494                                         NULL_TREE);
10495       decl = add_builtin_function ("__builtin_fabsq", ftype,
10496                                    IA64_BUILTIN_FABSQ, BUILT_IN_MD,
10497                                    "__fabstf2", NULL_TREE);
10498       TREE_READONLY (decl) = 1;
10499       ia64_builtins[IA64_BUILTIN_FABSQ] = decl;
10500
10501       ftype = build_function_type_list (float128_type_node,
10502                                         float128_type_node,
10503                                         float128_type_node,
10504                                         NULL_TREE);
10505       decl = add_builtin_function ("__builtin_copysignq", ftype,
10506                                    IA64_BUILTIN_COPYSIGNQ, BUILT_IN_MD,
10507                                    "__copysigntf3", NULL_TREE);
10508       TREE_READONLY (decl) = 1;
10509       ia64_builtins[IA64_BUILTIN_COPYSIGNQ] = decl;
10510     }
10511   else
10512     /* Under HPUX, this is a synonym for "long double".  */
10513     (*lang_hooks.types.register_builtin_type) (long_double_type_node,
10514                                                "__float128");
10515
10516   /* Fwrite on VMS is non-standard.  */
10517 #if TARGET_ABI_OPEN_VMS
10518   vms_patch_builtins ();
10519 #endif
10520
10521 #define def_builtin(name, type, code)                                   \
10522   add_builtin_function ((name), (type), (code), BUILT_IN_MD,    \
10523                        NULL, NULL_TREE)
10524
10525   decl = def_builtin ("__builtin_ia64_bsp",
10526                       build_function_type_list (ptr_type_node, NULL_TREE),
10527                       IA64_BUILTIN_BSP);
10528   ia64_builtins[IA64_BUILTIN_BSP] = decl;
10529
10530   decl = def_builtin ("__builtin_ia64_flushrs",
10531                       build_function_type_list (void_type_node, NULL_TREE),
10532                       IA64_BUILTIN_FLUSHRS);
10533   ia64_builtins[IA64_BUILTIN_FLUSHRS] = decl;
10534
10535 #undef def_builtin
10536
10537   if (TARGET_HPUX)
10538     {
10539       if ((decl = builtin_decl_explicit (BUILT_IN_FINITE)) != NULL_TREE)
10540         set_user_assembler_name (decl, "_Isfinite");
10541       if ((decl = builtin_decl_explicit (BUILT_IN_FINITEF)) != NULL_TREE)
10542         set_user_assembler_name (decl, "_Isfinitef");
10543       if ((decl = builtin_decl_explicit (BUILT_IN_FINITEL)) != NULL_TREE)
10544         set_user_assembler_name (decl, "_Isfinitef128");
10545     }
10546 }
10547
10548 static tree
10549 ia64_fold_builtin (tree fndecl, int n_args ATTRIBUTE_UNUSED,
10550                    tree *args, bool ignore ATTRIBUTE_UNUSED)
10551 {
10552   if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
10553     {
10554       enum ia64_builtins fn_code
10555         = (enum ia64_builtins) DECL_MD_FUNCTION_CODE (fndecl);
10556       switch (fn_code)
10557         {
10558         case IA64_BUILTIN_NANQ:
10559         case IA64_BUILTIN_NANSQ:
10560           {
10561             tree type = TREE_TYPE (TREE_TYPE (fndecl));
10562             const char *str = c_getstr (*args);
10563             int quiet = fn_code == IA64_BUILTIN_NANQ;
10564             REAL_VALUE_TYPE real;
10565
10566             if (str && real_nan (&real, str, quiet, TYPE_MODE (type)))
10567               return build_real (type, real);
10568             return NULL_TREE;
10569           }
10570
10571         default:
10572           break;
10573         }
10574     }
10575
10576 #ifdef SUBTARGET_FOLD_BUILTIN
10577   return SUBTARGET_FOLD_BUILTIN (fndecl, n_args, args, ignore);
10578 #endif
10579
10580   return NULL_TREE;
10581 }
10582
10583 rtx
10584 ia64_expand_builtin (tree exp, rtx target, rtx subtarget ATTRIBUTE_UNUSED,
10585                      machine_mode mode ATTRIBUTE_UNUSED,
10586                      int ignore ATTRIBUTE_UNUSED)
10587 {
10588   tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
10589   unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl);
10590
10591   switch (fcode)
10592     {
10593     case IA64_BUILTIN_BSP:
10594       if (! target || ! register_operand (target, DImode))
10595         target = gen_reg_rtx (DImode);
10596       emit_insn (gen_bsp_value (target));
10597 #ifdef POINTERS_EXTEND_UNSIGNED
10598       target = convert_memory_address (ptr_mode, target);
10599 #endif
10600       return target;
10601
10602     case IA64_BUILTIN_FLUSHRS:
10603       emit_insn (gen_flushrs ());
10604       return const0_rtx;
10605
10606     case IA64_BUILTIN_INFQ:
10607     case IA64_BUILTIN_HUGE_VALQ:
10608       {
10609         machine_mode target_mode = TYPE_MODE (TREE_TYPE (exp));
10610         REAL_VALUE_TYPE inf;
10611         rtx tmp;
10612
10613         real_inf (&inf);
10614         tmp = const_double_from_real_value (inf, target_mode);
10615
10616         tmp = validize_mem (force_const_mem (target_mode, tmp));
10617
10618         if (target == 0)
10619           target = gen_reg_rtx (target_mode);
10620
10621         emit_move_insn (target, tmp);
10622         return target;
10623       }
10624
10625     case IA64_BUILTIN_NANQ:
10626     case IA64_BUILTIN_NANSQ:
10627     case IA64_BUILTIN_FABSQ:
10628     case IA64_BUILTIN_COPYSIGNQ:
10629       return expand_call (exp, target, ignore);
10630
10631     default:
10632       gcc_unreachable ();
10633     }
10634
10635   return NULL_RTX;
10636 }
10637
10638 /* Return the ia64 builtin for CODE.  */
10639
10640 static tree
10641 ia64_builtin_decl (unsigned code, bool initialize_p ATTRIBUTE_UNUSED)
10642 {
10643   if (code >= IA64_BUILTIN_max)
10644     return error_mark_node;
10645
10646   return ia64_builtins[code];
10647 }
10648
10649 /* Implement TARGET_FUNCTION_ARG_PADDING.
10650
10651    For the HP-UX IA64 aggregate parameters are passed stored in the
10652    most significant bits of the stack slot.  */
10653
10654 static pad_direction
10655 ia64_function_arg_padding (machine_mode mode, const_tree type)
10656 {
10657   /* Exception to normal case for structures/unions/etc.  */
10658   if (TARGET_HPUX
10659       && type
10660       && AGGREGATE_TYPE_P (type)
10661       && int_size_in_bytes (type) < UNITS_PER_WORD)
10662     return PAD_UPWARD;
10663
10664   /* Fall back to the default.  */
10665   return default_function_arg_padding (mode, type);
10666 }
10667
10668 /* Emit text to declare externally defined variables and functions, because
10669    the Intel assembler does not support undefined externals.  */
10670
10671 void
10672 ia64_asm_output_external (FILE *file, tree decl, const char *name)
10673 {
10674   /* We output the name if and only if TREE_SYMBOL_REFERENCED is
10675      set in order to avoid putting out names that are never really
10676      used. */
10677   if (TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (decl)))
10678     {
10679       /* maybe_assemble_visibility will return 1 if the assembler
10680          visibility directive is output.  */
10681       int need_visibility = ((*targetm.binds_local_p) (decl)
10682                              && maybe_assemble_visibility (decl));
10683
10684       /* GNU as does not need anything here, but the HP linker does
10685          need something for external functions.  */
10686       if ((TARGET_HPUX_LD || !TARGET_GNU_AS)
10687           && TREE_CODE (decl) == FUNCTION_DECL)
10688           (*targetm.asm_out.globalize_decl_name) (file, decl);
10689       else if (need_visibility && !TARGET_GNU_AS)
10690         (*targetm.asm_out.globalize_label) (file, name);
10691     }
10692 }
10693
10694 /* Set SImode div/mod functions, init_integral_libfuncs only initializes
10695    modes of word_mode and larger.  Rename the TFmode libfuncs using the
10696    HPUX conventions. __divtf3 is used for XFmode. We need to keep it for
10697    backward compatibility. */
10698
10699 static void
10700 ia64_init_libfuncs (void)
10701 {
10702   set_optab_libfunc (sdiv_optab, SImode, "__divsi3");
10703   set_optab_libfunc (udiv_optab, SImode, "__udivsi3");
10704   set_optab_libfunc (smod_optab, SImode, "__modsi3");
10705   set_optab_libfunc (umod_optab, SImode, "__umodsi3");
10706
10707   set_optab_libfunc (add_optab, TFmode, "_U_Qfadd");
10708   set_optab_libfunc (sub_optab, TFmode, "_U_Qfsub");
10709   set_optab_libfunc (smul_optab, TFmode, "_U_Qfmpy");
10710   set_optab_libfunc (sdiv_optab, TFmode, "_U_Qfdiv");
10711   set_optab_libfunc (neg_optab, TFmode, "_U_Qfneg");
10712
10713   set_conv_libfunc (sext_optab, TFmode, SFmode, "_U_Qfcnvff_sgl_to_quad");
10714   set_conv_libfunc (sext_optab, TFmode, DFmode, "_U_Qfcnvff_dbl_to_quad");
10715   set_conv_libfunc (sext_optab, TFmode, XFmode, "_U_Qfcnvff_f80_to_quad");
10716   set_conv_libfunc (trunc_optab, SFmode, TFmode, "_U_Qfcnvff_quad_to_sgl");
10717   set_conv_libfunc (trunc_optab, DFmode, TFmode, "_U_Qfcnvff_quad_to_dbl");
10718   set_conv_libfunc (trunc_optab, XFmode, TFmode, "_U_Qfcnvff_quad_to_f80");
10719
10720   set_conv_libfunc (sfix_optab, SImode, TFmode, "_U_Qfcnvfxt_quad_to_sgl");
10721   set_conv_libfunc (sfix_optab, DImode, TFmode, "_U_Qfcnvfxt_quad_to_dbl");
10722   set_conv_libfunc (sfix_optab, TImode, TFmode, "_U_Qfcnvfxt_quad_to_quad");
10723   set_conv_libfunc (ufix_optab, SImode, TFmode, "_U_Qfcnvfxut_quad_to_sgl");
10724   set_conv_libfunc (ufix_optab, DImode, TFmode, "_U_Qfcnvfxut_quad_to_dbl");
10725
10726   set_conv_libfunc (sfloat_optab, TFmode, SImode, "_U_Qfcnvxf_sgl_to_quad");
10727   set_conv_libfunc (sfloat_optab, TFmode, DImode, "_U_Qfcnvxf_dbl_to_quad");
10728   set_conv_libfunc (sfloat_optab, TFmode, TImode, "_U_Qfcnvxf_quad_to_quad");
10729   /* HP-UX 11.23 libc does not have a function for unsigned
10730      SImode-to-TFmode conversion.  */
10731   set_conv_libfunc (ufloat_optab, TFmode, DImode, "_U_Qfcnvxuf_dbl_to_quad");
10732 }
10733
10734 /* Rename all the TFmode libfuncs using the HPUX conventions.  */
10735
10736 static void
10737 ia64_hpux_init_libfuncs (void)
10738 {
10739   ia64_init_libfuncs ();
10740
10741   /* The HP SI millicode division and mod functions expect DI arguments.
10742      By turning them off completely we avoid using both libgcc and the
10743      non-standard millicode routines and use the HP DI millicode routines
10744      instead.  */
10745
10746   set_optab_libfunc (sdiv_optab, SImode, 0);
10747   set_optab_libfunc (udiv_optab, SImode, 0);
10748   set_optab_libfunc (smod_optab, SImode, 0);
10749   set_optab_libfunc (umod_optab, SImode, 0);
10750
10751   set_optab_libfunc (sdiv_optab, DImode, "__milli_divI");
10752   set_optab_libfunc (udiv_optab, DImode, "__milli_divU");
10753   set_optab_libfunc (smod_optab, DImode, "__milli_remI");
10754   set_optab_libfunc (umod_optab, DImode, "__milli_remU");
10755
10756   /* HP-UX libc has TF min/max/abs routines in it.  */
10757   set_optab_libfunc (smin_optab, TFmode, "_U_Qfmin");
10758   set_optab_libfunc (smax_optab, TFmode, "_U_Qfmax");
10759   set_optab_libfunc (abs_optab, TFmode, "_U_Qfabs");
10760
10761   /* ia64_expand_compare uses this.  */
10762   cmptf_libfunc = init_one_libfunc ("_U_Qfcmp");
10763
10764   /* These should never be used.  */
10765   set_optab_libfunc (eq_optab, TFmode, 0);
10766   set_optab_libfunc (ne_optab, TFmode, 0);
10767   set_optab_libfunc (gt_optab, TFmode, 0);
10768   set_optab_libfunc (ge_optab, TFmode, 0);
10769   set_optab_libfunc (lt_optab, TFmode, 0);
10770   set_optab_libfunc (le_optab, TFmode, 0);
10771 }
10772
10773 /* Rename the division and modulus functions in VMS.  */
10774
10775 static void
10776 ia64_vms_init_libfuncs (void)
10777 {
10778   set_optab_libfunc (sdiv_optab, SImode, "OTS$DIV_I");
10779   set_optab_libfunc (sdiv_optab, DImode, "OTS$DIV_L");
10780   set_optab_libfunc (udiv_optab, SImode, "OTS$DIV_UI");
10781   set_optab_libfunc (udiv_optab, DImode, "OTS$DIV_UL");
10782   set_optab_libfunc (smod_optab, SImode, "OTS$REM_I");
10783   set_optab_libfunc (smod_optab, DImode, "OTS$REM_L");
10784   set_optab_libfunc (umod_optab, SImode, "OTS$REM_UI");
10785   set_optab_libfunc (umod_optab, DImode, "OTS$REM_UL");
10786 #ifdef MEM_LIBFUNCS_INIT
10787   MEM_LIBFUNCS_INIT;
10788 #endif
10789 }
10790
10791 /* Rename the TFmode libfuncs available from soft-fp in glibc using
10792    the HPUX conventions.  */
10793
10794 static void
10795 ia64_sysv4_init_libfuncs (void)
10796 {
10797   ia64_init_libfuncs ();
10798
10799   /* These functions are not part of the HPUX TFmode interface.  We
10800      use them instead of _U_Qfcmp, which doesn't work the way we
10801      expect.  */
10802   set_optab_libfunc (eq_optab, TFmode, "_U_Qfeq");
10803   set_optab_libfunc (ne_optab, TFmode, "_U_Qfne");
10804   set_optab_libfunc (gt_optab, TFmode, "_U_Qfgt");
10805   set_optab_libfunc (ge_optab, TFmode, "_U_Qfge");
10806   set_optab_libfunc (lt_optab, TFmode, "_U_Qflt");
10807   set_optab_libfunc (le_optab, TFmode, "_U_Qfle");
10808
10809   /* We leave out _U_Qfmin, _U_Qfmax and _U_Qfabs since soft-fp in
10810      glibc doesn't have them.  */
10811 }
10812
10813 /* Use soft-fp.  */
10814
10815 static void
10816 ia64_soft_fp_init_libfuncs (void)
10817 {
10818 }
10819
10820 static bool
10821 ia64_vms_valid_pointer_mode (scalar_int_mode mode)
10822 {
10823   return (mode == SImode || mode == DImode);
10824 }
10825 \f
10826 /* For HPUX, it is illegal to have relocations in shared segments.  */
10827
10828 static int
10829 ia64_hpux_reloc_rw_mask (void)
10830 {
10831   return 3;
10832 }
10833
10834 /* For others, relax this so that relocations to local data goes in
10835    read-only segments, but we still cannot allow global relocations
10836    in read-only segments.  */
10837
10838 static int
10839 ia64_reloc_rw_mask (void)
10840 {
10841   return flag_pic ? 3 : 2;
10842 }
10843
10844 /* Return the section to use for X.  The only special thing we do here
10845    is to honor small data.  */
10846
10847 static section *
10848 ia64_select_rtx_section (machine_mode mode, rtx x,
10849                          unsigned HOST_WIDE_INT align)
10850 {
10851   if (GET_MODE_SIZE (mode) > 0
10852       && GET_MODE_SIZE (mode) <= ia64_section_threshold
10853       && !TARGET_NO_SDATA)
10854     return sdata_section;
10855   else
10856     return default_elf_select_rtx_section (mode, x, align);
10857 }
10858
10859 static unsigned int
10860 ia64_section_type_flags (tree decl, const char *name, int reloc)
10861 {
10862   unsigned int flags = 0;
10863
10864   if (strcmp (name, ".sdata") == 0
10865       || strncmp (name, ".sdata.", 7) == 0
10866       || strncmp (name, ".gnu.linkonce.s.", 16) == 0
10867       || strncmp (name, ".sdata2.", 8) == 0
10868       || strncmp (name, ".gnu.linkonce.s2.", 17) == 0
10869       || strcmp (name, ".sbss") == 0
10870       || strncmp (name, ".sbss.", 6) == 0
10871       || strncmp (name, ".gnu.linkonce.sb.", 17) == 0)
10872     flags = SECTION_SMALL;
10873
10874   flags |= default_section_type_flags (decl, name, reloc);
10875   return flags;
10876 }
10877
10878 /* Returns true if FNTYPE (a FUNCTION_TYPE or a METHOD_TYPE) returns a
10879    structure type and that the address of that type should be passed
10880    in out0, rather than in r8.  */
10881
10882 static bool
10883 ia64_struct_retval_addr_is_first_parm_p (tree fntype)
10884 {
10885   tree ret_type = TREE_TYPE (fntype);
10886
10887   /* The Itanium C++ ABI requires that out0, rather than r8, be used
10888      as the structure return address parameter, if the return value
10889      type has a non-trivial copy constructor or destructor.  It is not
10890      clear if this same convention should be used for other
10891      programming languages.  Until G++ 3.4, we incorrectly used r8 for
10892      these return values.  */
10893   return (abi_version_at_least (2)
10894           && ret_type
10895           && TYPE_MODE (ret_type) == BLKmode
10896           && TREE_ADDRESSABLE (ret_type)
10897           && lang_GNU_CXX ());
10898 }
10899
10900 /* Output the assembler code for a thunk function.  THUNK_DECL is the
10901    declaration for the thunk function itself, FUNCTION is the decl for
10902    the target function.  DELTA is an immediate constant offset to be
10903    added to THIS.  If VCALL_OFFSET is nonzero, the word at
10904    *(*this + vcall_offset) should be added to THIS.  */
10905
10906 static void
10907 ia64_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED,
10908                       HOST_WIDE_INT delta, HOST_WIDE_INT vcall_offset,
10909                       tree function)
10910 {
10911   const char *fnname = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (thunk));
10912   rtx this_rtx, funexp;
10913   rtx_insn *insn;
10914   unsigned int this_parmno;
10915   unsigned int this_regno;
10916   rtx delta_rtx;
10917
10918   reload_completed = 1;
10919   epilogue_completed = 1;
10920
10921   /* Set things up as ia64_expand_prologue might.  */
10922   last_scratch_gr_reg = 15;
10923
10924   memset (&current_frame_info, 0, sizeof (current_frame_info));
10925   current_frame_info.spill_cfa_off = -16;
10926   current_frame_info.n_input_regs = 1;
10927   current_frame_info.need_regstk = (TARGET_REG_NAMES != 0);
10928
10929   /* Mark the end of the (empty) prologue.  */
10930   emit_note (NOTE_INSN_PROLOGUE_END);
10931
10932   /* Figure out whether "this" will be the first parameter (the
10933      typical case) or the second parameter (as happens when the
10934      virtual function returns certain class objects).  */
10935   this_parmno
10936     = (ia64_struct_retval_addr_is_first_parm_p (TREE_TYPE (thunk))
10937        ? 1 : 0);
10938   this_regno = IN_REG (this_parmno);
10939   if (!TARGET_REG_NAMES)
10940     reg_names[this_regno] = ia64_reg_numbers[this_parmno];
10941
10942   this_rtx = gen_rtx_REG (Pmode, this_regno);
10943
10944   /* Apply the constant offset, if required.  */
10945   delta_rtx = GEN_INT (delta);
10946   if (TARGET_ILP32)
10947     {
10948       rtx tmp = gen_rtx_REG (ptr_mode, this_regno);
10949       REG_POINTER (tmp) = 1;
10950       if (delta && satisfies_constraint_I (delta_rtx))
10951         {
10952           emit_insn (gen_ptr_extend_plus_imm (this_rtx, tmp, delta_rtx));
10953           delta = 0;
10954         }
10955       else
10956         emit_insn (gen_ptr_extend (this_rtx, tmp));
10957     }
10958   if (delta)
10959     {
10960       if (!satisfies_constraint_I (delta_rtx))
10961         {
10962           rtx tmp = gen_rtx_REG (Pmode, 2);
10963           emit_move_insn (tmp, delta_rtx);
10964           delta_rtx = tmp;
10965         }
10966       emit_insn (gen_adddi3 (this_rtx, this_rtx, delta_rtx));
10967     }
10968
10969   /* Apply the offset from the vtable, if required.  */
10970   if (vcall_offset)
10971     {
10972       rtx vcall_offset_rtx = GEN_INT (vcall_offset);
10973       rtx tmp = gen_rtx_REG (Pmode, 2);
10974
10975       if (TARGET_ILP32)
10976         {
10977           rtx t = gen_rtx_REG (ptr_mode, 2);
10978           REG_POINTER (t) = 1;
10979           emit_move_insn (t, gen_rtx_MEM (ptr_mode, this_rtx));
10980           if (satisfies_constraint_I (vcall_offset_rtx))
10981             {
10982               emit_insn (gen_ptr_extend_plus_imm (tmp, t, vcall_offset_rtx));
10983               vcall_offset = 0;
10984             }
10985           else
10986             emit_insn (gen_ptr_extend (tmp, t));
10987         }
10988       else
10989         emit_move_insn (tmp, gen_rtx_MEM (Pmode, this_rtx));
10990
10991       if (vcall_offset)
10992         {
10993           if (!satisfies_constraint_J (vcall_offset_rtx))
10994             {
10995               rtx tmp2 = gen_rtx_REG (Pmode, next_scratch_gr_reg ());
10996               emit_move_insn (tmp2, vcall_offset_rtx);
10997               vcall_offset_rtx = tmp2;
10998             }
10999           emit_insn (gen_adddi3 (tmp, tmp, vcall_offset_rtx));
11000         }
11001
11002       if (TARGET_ILP32)
11003         emit_insn (gen_zero_extendsidi2 (tmp, gen_rtx_MEM (ptr_mode, tmp)));
11004       else
11005         emit_move_insn (tmp, gen_rtx_MEM (Pmode, tmp));
11006
11007       emit_insn (gen_adddi3 (this_rtx, this_rtx, tmp));
11008     }
11009
11010   /* Generate a tail call to the target function.  */
11011   if (! TREE_USED (function))
11012     {
11013       assemble_external (function);
11014       TREE_USED (function) = 1;
11015     }
11016   funexp = XEXP (DECL_RTL (function), 0);
11017   funexp = gen_rtx_MEM (FUNCTION_MODE, funexp);
11018   ia64_expand_call (NULL_RTX, funexp, NULL_RTX, 1);
11019   insn = get_last_insn ();
11020   SIBLING_CALL_P (insn) = 1;
11021
11022   /* Code generation for calls relies on splitting.  */
11023   reload_completed = 1;
11024   epilogue_completed = 1;
11025   try_split (PATTERN (insn), insn, 0);
11026
11027   emit_barrier ();
11028
11029   /* Run just enough of rest_of_compilation to get the insns emitted.
11030      There's not really enough bulk here to make other passes such as
11031      instruction scheduling worth while.  */
11032
11033   emit_all_insn_group_barriers (NULL);
11034   insn = get_insns ();
11035   shorten_branches (insn);
11036   assemble_start_function (thunk, fnname);
11037   final_start_function (insn, file, 1);
11038   final (insn, file, 1);
11039   final_end_function ();
11040   assemble_end_function (thunk, fnname);
11041
11042   reload_completed = 0;
11043   epilogue_completed = 0;
11044 }
11045
11046 /* Worker function for TARGET_STRUCT_VALUE_RTX.  */
11047
11048 static rtx
11049 ia64_struct_value_rtx (tree fntype,
11050                        int incoming ATTRIBUTE_UNUSED)
11051 {
11052   if (TARGET_ABI_OPEN_VMS ||
11053       (fntype && ia64_struct_retval_addr_is_first_parm_p (fntype)))
11054     return NULL_RTX;
11055   return gen_rtx_REG (Pmode, GR_REG (8));
11056 }
11057
11058 static bool
11059 ia64_scalar_mode_supported_p (scalar_mode mode)
11060 {
11061   switch (mode)
11062     {
11063     case E_QImode:
11064     case E_HImode:
11065     case E_SImode:
11066     case E_DImode:
11067     case E_TImode:
11068       return true;
11069
11070     case E_SFmode:
11071     case E_DFmode:
11072     case E_XFmode:
11073     case E_RFmode:
11074       return true;
11075
11076     case E_TFmode:
11077       return true;
11078
11079     default:
11080       return false;
11081     }
11082 }
11083
11084 static bool
11085 ia64_vector_mode_supported_p (machine_mode mode)
11086 {
11087   switch (mode)
11088     {
11089     case E_V8QImode:
11090     case E_V4HImode:
11091     case E_V2SImode:
11092       return true;
11093
11094     case E_V2SFmode:
11095       return true;
11096
11097     default:
11098       return false;
11099     }
11100 }
11101
11102 /* Implement the FUNCTION_PROFILER macro.  */
11103
11104 void
11105 ia64_output_function_profiler (FILE *file, int labelno)
11106 {
11107   bool indirect_call;
11108
11109   /* If the function needs a static chain and the static chain
11110      register is r15, we use an indirect call so as to bypass
11111      the PLT stub in case the executable is dynamically linked,
11112      because the stub clobbers r15 as per 5.3.6 of the psABI.
11113      We don't need to do that in non canonical PIC mode.  */
11114
11115   if (cfun->static_chain_decl && !TARGET_NO_PIC && !TARGET_AUTO_PIC)
11116     {
11117       gcc_assert (STATIC_CHAIN_REGNUM == 15);
11118       indirect_call = true;
11119     }
11120   else
11121     indirect_call = false;
11122
11123   if (TARGET_GNU_AS)
11124     fputs ("\t.prologue 4, r40\n", file);
11125   else
11126     fputs ("\t.prologue\n\t.save ar.pfs, r40\n", file);
11127   fputs ("\talloc out0 = ar.pfs, 8, 0, 4, 0\n", file);
11128
11129   if (NO_PROFILE_COUNTERS)
11130     fputs ("\tmov out3 = r0\n", file);
11131   else
11132     {
11133       char buf[20];
11134       ASM_GENERATE_INTERNAL_LABEL (buf, "LP", labelno);
11135
11136       if (TARGET_AUTO_PIC)
11137         fputs ("\tmovl out3 = @gprel(", file);
11138       else
11139         fputs ("\taddl out3 = @ltoff(", file);
11140       assemble_name (file, buf);
11141       if (TARGET_AUTO_PIC)
11142         fputs (")\n", file);
11143       else
11144         fputs ("), r1\n", file);
11145     }
11146
11147   if (indirect_call)
11148     fputs ("\taddl r14 = @ltoff(@fptr(_mcount)), r1\n", file);
11149   fputs ("\t;;\n", file);
11150
11151   fputs ("\t.save rp, r42\n", file);
11152   fputs ("\tmov out2 = b0\n", file);
11153   if (indirect_call)
11154     fputs ("\tld8 r14 = [r14]\n\t;;\n", file);
11155   fputs ("\t.body\n", file);
11156   fputs ("\tmov out1 = r1\n", file);
11157   if (indirect_call)
11158     {
11159       fputs ("\tld8 r16 = [r14], 8\n\t;;\n", file);
11160       fputs ("\tmov b6 = r16\n", file);
11161       fputs ("\tld8 r1 = [r14]\n", file);
11162       fputs ("\tbr.call.sptk.many b0 = b6\n\t;;\n", file);
11163     }
11164   else
11165     fputs ("\tbr.call.sptk.many b0 = _mcount\n\t;;\n", file);
11166 }
11167
11168 static GTY(()) rtx mcount_func_rtx;
11169 static rtx
11170 gen_mcount_func_rtx (void)
11171 {
11172   if (!mcount_func_rtx)
11173     mcount_func_rtx = init_one_libfunc ("_mcount");
11174   return mcount_func_rtx;
11175 }
11176
11177 void
11178 ia64_profile_hook (int labelno)
11179 {
11180   rtx label, ip;
11181
11182   if (NO_PROFILE_COUNTERS)
11183     label = const0_rtx;
11184   else
11185     {
11186       char buf[30];
11187       const char *label_name;
11188       ASM_GENERATE_INTERNAL_LABEL (buf, "LP", labelno);
11189       label_name = ggc_strdup ((*targetm.strip_name_encoding) (buf));
11190       label = gen_rtx_SYMBOL_REF (Pmode, label_name);
11191       SYMBOL_REF_FLAGS (label) = SYMBOL_FLAG_LOCAL;
11192     }
11193   ip = gen_reg_rtx (Pmode);
11194   emit_insn (gen_ip_value (ip));
11195   emit_library_call (gen_mcount_func_rtx (), LCT_NORMAL,
11196                      VOIDmode,
11197                      gen_rtx_REG (Pmode, BR_REG (0)), Pmode,
11198                      ip, Pmode,
11199                      label, Pmode);
11200 }
11201
11202 /* Return the mangling of TYPE if it is an extended fundamental type.  */
11203
11204 static const char *
11205 ia64_mangle_type (const_tree type)
11206 {
11207   type = TYPE_MAIN_VARIANT (type);
11208
11209   if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
11210       && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
11211     return NULL;
11212
11213   /* On HP-UX, "long double" is mangled as "e" so __float128 is
11214      mangled as "e".  */
11215   if (!TARGET_HPUX && TYPE_MODE (type) == TFmode)
11216     return "g";
11217   /* On HP-UX, "e" is not available as a mangling of __float80 so use
11218      an extended mangling.  Elsewhere, "e" is available since long
11219      double is 80 bits.  */
11220   if (TYPE_MODE (type) == XFmode)
11221     return TARGET_HPUX ? "u9__float80" : "e";
11222   if (TYPE_MODE (type) == RFmode)
11223     return "u7__fpreg";
11224   return NULL;
11225 }
11226
11227 /* Return the diagnostic message string if conversion from FROMTYPE to
11228    TOTYPE is not allowed, NULL otherwise.  */
11229 static const char *
11230 ia64_invalid_conversion (const_tree fromtype, const_tree totype)
11231 {
11232   /* Reject nontrivial conversion to or from __fpreg.  */
11233   if (TYPE_MODE (fromtype) == RFmode
11234       && TYPE_MODE (totype) != RFmode
11235       && TYPE_MODE (totype) != VOIDmode)
11236     return N_("invalid conversion from %<__fpreg%>");
11237   if (TYPE_MODE (totype) == RFmode
11238       && TYPE_MODE (fromtype) != RFmode)
11239     return N_("invalid conversion to %<__fpreg%>");
11240   return NULL;
11241 }
11242
11243 /* Return the diagnostic message string if the unary operation OP is
11244    not permitted on TYPE, NULL otherwise.  */
11245 static const char *
11246 ia64_invalid_unary_op (int op, const_tree type)
11247 {
11248   /* Reject operations on __fpreg other than unary + or &.  */
11249   if (TYPE_MODE (type) == RFmode
11250       && op != CONVERT_EXPR
11251       && op != ADDR_EXPR)
11252     return N_("invalid operation on %<__fpreg%>");
11253   return NULL;
11254 }
11255
11256 /* Return the diagnostic message string if the binary operation OP is
11257    not permitted on TYPE1 and TYPE2, NULL otherwise.  */
11258 static const char *
11259 ia64_invalid_binary_op (int op ATTRIBUTE_UNUSED, const_tree type1, const_tree type2)
11260 {
11261   /* Reject operations on __fpreg.  */
11262   if (TYPE_MODE (type1) == RFmode || TYPE_MODE (type2) == RFmode)
11263     return N_("invalid operation on %<__fpreg%>");
11264   return NULL;
11265 }
11266
11267 /* HP-UX version_id attribute.
11268    For object foo, if the version_id is set to 1234 put out an alias
11269    of '.alias foo "foo{1234}"  We can't use "foo{1234}" in anything
11270    other than an alias statement because it is an illegal symbol name.  */
11271
11272 static tree
11273 ia64_handle_version_id_attribute (tree *node ATTRIBUTE_UNUSED,
11274                                  tree name ATTRIBUTE_UNUSED,
11275                                  tree args,
11276                                  int flags ATTRIBUTE_UNUSED,
11277                                  bool *no_add_attrs)
11278 {
11279   tree arg = TREE_VALUE (args);
11280
11281   if (TREE_CODE (arg) != STRING_CST)
11282     {
11283       error("version attribute is not a string");
11284       *no_add_attrs = true;
11285       return NULL_TREE;
11286     }
11287   return NULL_TREE;
11288 }
11289
11290 /* Target hook for c_mode_for_suffix.  */
11291
11292 static machine_mode
11293 ia64_c_mode_for_suffix (char suffix)
11294 {
11295   if (suffix == 'q')
11296     return TFmode;
11297   if (suffix == 'w')
11298     return XFmode;
11299
11300   return VOIDmode;
11301 }
11302
11303 static GTY(()) rtx ia64_dconst_0_5_rtx;
11304
11305 rtx
11306 ia64_dconst_0_5 (void)
11307 {
11308   if (! ia64_dconst_0_5_rtx)
11309     {
11310       REAL_VALUE_TYPE rv;
11311       real_from_string (&rv, "0.5");
11312       ia64_dconst_0_5_rtx = const_double_from_real_value (rv, DFmode);
11313     }
11314   return ia64_dconst_0_5_rtx;
11315 }
11316
11317 static GTY(()) rtx ia64_dconst_0_375_rtx;
11318
11319 rtx
11320 ia64_dconst_0_375 (void)
11321 {
11322   if (! ia64_dconst_0_375_rtx)
11323     {
11324       REAL_VALUE_TYPE rv;
11325       real_from_string (&rv, "0.375");
11326       ia64_dconst_0_375_rtx = const_double_from_real_value (rv, DFmode);
11327     }
11328   return ia64_dconst_0_375_rtx;
11329 }
11330
11331 static fixed_size_mode
11332 ia64_get_reg_raw_mode (int regno)
11333 {
11334   if (FR_REGNO_P (regno))
11335     return XFmode;
11336   return default_get_reg_raw_mode(regno);
11337 }
11338
11339 /* Implement TARGET_MEMBER_TYPE_FORCES_BLK.  ??? Might not be needed
11340    anymore.  */
11341
11342 bool
11343 ia64_member_type_forces_blk (const_tree, machine_mode mode)
11344 {
11345   return TARGET_HPUX && mode == TFmode;
11346 }
11347
11348 /* Always default to .text section until HP-UX linker is fixed.  */
11349
11350 ATTRIBUTE_UNUSED static section *
11351 ia64_hpux_function_section (tree decl ATTRIBUTE_UNUSED,
11352                             enum node_frequency freq ATTRIBUTE_UNUSED,
11353                             bool startup ATTRIBUTE_UNUSED,
11354                             bool exit ATTRIBUTE_UNUSED)
11355 {
11356   return NULL;
11357 }
11358 \f
11359 /* Construct (set target (vec_select op0 (parallel perm))) and
11360    return true if that's a valid instruction in the active ISA.  */
11361
11362 static bool
11363 expand_vselect (rtx target, rtx op0, const unsigned char *perm, unsigned nelt)
11364 {
11365   rtx rperm[MAX_VECT_LEN], x;
11366   unsigned i;
11367
11368   for (i = 0; i < nelt; ++i)
11369     rperm[i] = GEN_INT (perm[i]);
11370
11371   x = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelt, rperm));
11372   x = gen_rtx_VEC_SELECT (GET_MODE (target), op0, x);
11373   x = gen_rtx_SET (target, x);
11374
11375   rtx_insn *insn = emit_insn (x);
11376   if (recog_memoized (insn) < 0)
11377     {
11378       remove_insn (insn);
11379       return false;
11380     }
11381   return true;
11382 }
11383
11384 /* Similar, but generate a vec_concat from op0 and op1 as well.  */
11385
11386 static bool
11387 expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
11388                         const unsigned char *perm, unsigned nelt)
11389 {
11390   machine_mode v2mode;
11391   rtx x;
11392
11393   if (!GET_MODE_2XWIDER_MODE (GET_MODE (op0)).exists (&v2mode))
11394     return false;
11395   x = gen_rtx_VEC_CONCAT (v2mode, op0, op1);
11396   return expand_vselect (target, x, perm, nelt);
11397 }
11398
11399 /* Try to expand a no-op permutation.  */
11400
11401 static bool
11402 expand_vec_perm_identity (struct expand_vec_perm_d *d)
11403 {
11404   unsigned i, nelt = d->nelt;
11405
11406   for (i = 0; i < nelt; ++i)
11407     if (d->perm[i] != i)
11408       return false;
11409
11410   if (!d->testing_p)
11411     emit_move_insn (d->target, d->op0);
11412
11413   return true;
11414 }
11415
11416 /* Try to expand D via a shrp instruction.  */
11417
11418 static bool
11419 expand_vec_perm_shrp (struct expand_vec_perm_d *d)
11420 {
11421   unsigned i, nelt = d->nelt, shift, mask;
11422   rtx tmp, hi, lo;
11423
11424   /* ??? Don't force V2SFmode into the integer registers.  */
11425   if (d->vmode == V2SFmode)
11426     return false;
11427
11428   mask = (d->one_operand_p ? nelt - 1 : 2 * nelt - 1);
11429
11430   shift = d->perm[0];
11431   if (BYTES_BIG_ENDIAN && shift > nelt)
11432     return false;
11433
11434   for (i = 1; i < nelt; ++i)
11435     if (d->perm[i] != ((shift + i) & mask))
11436       return false;
11437
11438   if (d->testing_p)
11439     return true;
11440
11441   hi = shift < nelt ? d->op1 : d->op0;
11442   lo = shift < nelt ? d->op0 : d->op1;
11443
11444   shift %= nelt;
11445
11446   shift *= GET_MODE_UNIT_SIZE (d->vmode) * BITS_PER_UNIT;
11447
11448   /* We've eliminated the shift 0 case via expand_vec_perm_identity.  */
11449   gcc_assert (IN_RANGE (shift, 1, 63));
11450
11451   /* Recall that big-endian elements are numbered starting at the top of
11452      the register.  Ideally we'd have a shift-left-pair.  But since we
11453      don't, convert to a shift the other direction.  */
11454   if (BYTES_BIG_ENDIAN)
11455     shift = 64 - shift;
11456
11457   tmp = gen_reg_rtx (DImode);
11458   hi = gen_lowpart (DImode, hi);
11459   lo = gen_lowpart (DImode, lo);
11460   emit_insn (gen_shrp (tmp, hi, lo, GEN_INT (shift)));
11461
11462   emit_move_insn (d->target, gen_lowpart (d->vmode, tmp));
11463   return true;
11464 }
11465
11466 /* Try to instantiate D in a single instruction.  */
11467
11468 static bool
11469 expand_vec_perm_1 (struct expand_vec_perm_d *d)
11470 {
11471   unsigned i, nelt = d->nelt;
11472   unsigned char perm2[MAX_VECT_LEN];
11473
11474   /* Try single-operand selections.  */
11475   if (d->one_operand_p)
11476     {
11477       if (expand_vec_perm_identity (d))
11478         return true;
11479       if (expand_vselect (d->target, d->op0, d->perm, nelt))
11480         return true;
11481     }
11482
11483   /* Try two operand selections.  */
11484   if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt))
11485     return true;
11486
11487   /* Recognize interleave style patterns with reversed operands.  */
11488   if (!d->one_operand_p)
11489     {
11490       for (i = 0; i < nelt; ++i)
11491         {
11492           unsigned e = d->perm[i];
11493           if (e >= nelt)
11494             e -= nelt;
11495           else
11496             e += nelt;
11497           perm2[i] = e;
11498         }
11499
11500       if (expand_vselect_vconcat (d->target, d->op1, d->op0, perm2, nelt))
11501         return true;
11502     }
11503
11504   if (expand_vec_perm_shrp (d))
11505     return true;
11506
11507   /* ??? Look for deposit-like permutations where most of the result
11508      comes from one vector unchanged and the rest comes from a
11509      sequential hunk of the other vector.  */
11510
11511   return false;
11512 }
11513
11514 /* Pattern match broadcast permutations.  */
11515
11516 static bool
11517 expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
11518 {
11519   unsigned i, elt, nelt = d->nelt;
11520   unsigned char perm2[2];
11521   rtx temp;
11522   bool ok;
11523
11524   if (!d->one_operand_p)
11525     return false;
11526
11527   elt = d->perm[0];
11528   for (i = 1; i < nelt; ++i)
11529     if (d->perm[i] != elt)
11530       return false;
11531
11532   switch (d->vmode)
11533     {
11534     case E_V2SImode:
11535     case E_V2SFmode:
11536       /* Implementable by interleave.  */
11537       perm2[0] = elt;
11538       perm2[1] = elt + 2;
11539       ok = expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, 2);
11540       gcc_assert (ok);
11541       break;
11542
11543     case E_V8QImode:
11544       /* Implementable by extract + broadcast.  */
11545       if (BYTES_BIG_ENDIAN)
11546         elt = 7 - elt;
11547       elt *= BITS_PER_UNIT;
11548       temp = gen_reg_rtx (DImode);
11549       emit_insn (gen_extzv (temp, gen_lowpart (DImode, d->op0),
11550                             GEN_INT (8), GEN_INT (elt)));
11551       emit_insn (gen_mux1_brcst_qi (d->target, gen_lowpart (QImode, temp)));
11552       break;
11553
11554     case E_V4HImode:
11555       /* Should have been matched directly by vec_select.  */
11556     default:
11557       gcc_unreachable ();
11558     }
11559
11560   return true;
11561 }
11562
11563 /* A subroutine of ia64_expand_vec_perm_const_1.  Try to simplify a
11564    two vector permutation into a single vector permutation by using
11565    an interleave operation to merge the vectors.  */
11566
11567 static bool
11568 expand_vec_perm_interleave_2 (struct expand_vec_perm_d *d)
11569 {
11570   struct expand_vec_perm_d dremap, dfinal;
11571   unsigned char remap[2 * MAX_VECT_LEN];
11572   unsigned contents, i, nelt, nelt2;
11573   unsigned h0, h1, h2, h3;
11574   rtx_insn *seq;
11575   bool ok;
11576
11577   if (d->one_operand_p)
11578     return false;
11579
11580   nelt = d->nelt;
11581   nelt2 = nelt / 2;
11582
11583   /* Examine from whence the elements come.  */
11584   contents = 0;
11585   for (i = 0; i < nelt; ++i)
11586     contents |= 1u << d->perm[i];
11587
11588   memset (remap, 0xff, sizeof (remap));
11589   dremap = *d;
11590
11591   h0 = (1u << nelt2) - 1;
11592   h1 = h0 << nelt2;
11593   h2 = h0 << nelt;
11594   h3 = h0 << (nelt + nelt2);
11595
11596   if ((contents & (h0 | h2)) == contents)       /* punpck even halves */
11597     {
11598       for (i = 0; i < nelt; ++i)
11599         {
11600           unsigned which = i / 2 + (i & 1 ? nelt : 0);
11601           remap[which] = i;
11602           dremap.perm[i] = which;
11603         }
11604     }
11605   else if ((contents & (h1 | h3)) == contents)  /* punpck odd halves */
11606     {
11607       for (i = 0; i < nelt; ++i)
11608         {
11609           unsigned which = i / 2 + nelt2 + (i & 1 ? nelt : 0);
11610           remap[which] = i;
11611           dremap.perm[i] = which;
11612         }
11613     }
11614   else if ((contents & 0x5555) == contents)     /* mix even elements */
11615     {
11616       for (i = 0; i < nelt; ++i)
11617         {
11618           unsigned which = (i & ~1) + (i & 1 ? nelt : 0);
11619           remap[which] = i;
11620           dremap.perm[i] = which;
11621         }
11622     }
11623   else if ((contents & 0xaaaa) == contents)     /* mix odd elements */
11624     {
11625       for (i = 0; i < nelt; ++i)
11626         {
11627           unsigned which = (i | 1) + (i & 1 ? nelt : 0);
11628           remap[which] = i;
11629           dremap.perm[i] = which;
11630         }
11631     }
11632   else if (floor_log2 (contents) - ctz_hwi (contents) < (int)nelt) /* shrp */
11633     {
11634       unsigned shift = ctz_hwi (contents);
11635       for (i = 0; i < nelt; ++i)
11636         {
11637           unsigned which = (i + shift) & (2 * nelt - 1);
11638           remap[which] = i;
11639           dremap.perm[i] = which;
11640         }
11641     }
11642   else
11643     return false;
11644
11645   /* Use the remapping array set up above to move the elements from their
11646      swizzled locations into their final destinations.  */
11647   dfinal = *d;
11648   for (i = 0; i < nelt; ++i)
11649     {
11650       unsigned e = remap[d->perm[i]];
11651       gcc_assert (e < nelt);
11652       dfinal.perm[i] = e;
11653     }
11654   if (d->testing_p)
11655     dfinal.op0 = gen_raw_REG (dfinal.vmode, LAST_VIRTUAL_REGISTER + 1);
11656   else
11657     dfinal.op0 = gen_reg_rtx (dfinal.vmode);
11658   dfinal.op1 = dfinal.op0;
11659   dfinal.one_operand_p = true;
11660   dremap.target = dfinal.op0;
11661
11662   /* Test if the final remap can be done with a single insn.  For V4HImode
11663      this *will* succeed.  For V8QImode or V2SImode it may not.  */
11664   start_sequence ();
11665   ok = expand_vec_perm_1 (&dfinal);
11666   seq = get_insns ();
11667   end_sequence ();
11668   if (!ok)
11669     return false;
11670   if (d->testing_p)
11671     return true;
11672
11673   ok = expand_vec_perm_1 (&dremap);
11674   gcc_assert (ok);
11675
11676   emit_insn (seq);
11677   return true;
11678 }
11679
11680 /* A subroutine of ia64_expand_vec_perm_const_1.  Emit a full V4HImode
11681    constant permutation via two mux2 and a merge.  */
11682
11683 static bool
11684 expand_vec_perm_v4hi_5 (struct expand_vec_perm_d *d)
11685 {
11686   unsigned char perm2[4];
11687   rtx rmask[4];
11688   unsigned i;
11689   rtx t0, t1, mask, x;
11690   bool ok;
11691
11692   if (d->vmode != V4HImode || d->one_operand_p)
11693     return false;
11694   if (d->testing_p)
11695     return true;
11696
11697   for (i = 0; i < 4; ++i)
11698     {
11699       perm2[i] = d->perm[i] & 3;
11700       rmask[i] = (d->perm[i] & 4 ? const0_rtx : constm1_rtx);
11701     }
11702   mask = gen_rtx_CONST_VECTOR (V4HImode, gen_rtvec_v (4, rmask));
11703   mask = force_reg (V4HImode, mask);
11704
11705   t0 = gen_reg_rtx (V4HImode);
11706   t1 = gen_reg_rtx (V4HImode);
11707
11708   ok = expand_vselect (t0, d->op0, perm2, 4);
11709   gcc_assert (ok);
11710   ok = expand_vselect (t1, d->op1, perm2, 4);
11711   gcc_assert (ok);
11712
11713   x = gen_rtx_AND (V4HImode, mask, t0);
11714   emit_insn (gen_rtx_SET (t0, x));
11715
11716   x = gen_rtx_NOT (V4HImode, mask);
11717   x = gen_rtx_AND (V4HImode, x, t1);
11718   emit_insn (gen_rtx_SET (t1, x));
11719
11720   x = gen_rtx_IOR (V4HImode, t0, t1);
11721   emit_insn (gen_rtx_SET (d->target, x));
11722
11723   return true;
11724 }
11725
11726 /* The guts of ia64_expand_vec_perm_const, also used by the ok hook.
11727    With all of the interface bits taken care of, perform the expansion
11728    in D and return true on success.  */
11729
11730 static bool
11731 ia64_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
11732 {
11733   if (expand_vec_perm_1 (d))
11734     return true;
11735   if (expand_vec_perm_broadcast (d))
11736     return true;
11737   if (expand_vec_perm_interleave_2 (d))
11738     return true;
11739   if (expand_vec_perm_v4hi_5 (d))
11740     return true;
11741   return false;
11742 }
11743
11744 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST.  */
11745
11746 static bool
11747 ia64_vectorize_vec_perm_const (machine_mode vmode, rtx target, rtx op0,
11748                                rtx op1, const vec_perm_indices &sel)
11749 {
11750   struct expand_vec_perm_d d;
11751   unsigned char perm[MAX_VECT_LEN];
11752   unsigned int i, nelt, which;
11753
11754   d.target = target;
11755   d.op0 = op0;
11756   d.op1 = op1;
11757
11758   d.vmode = vmode;
11759   gcc_assert (VECTOR_MODE_P (d.vmode));
11760   d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
11761   d.testing_p = !target;
11762
11763   gcc_assert (sel.length () == nelt);
11764   gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
11765
11766   for (i = which = 0; i < nelt; ++i)
11767     {
11768       unsigned int ei = sel[i] & (2 * nelt - 1);
11769
11770       which |= (ei < nelt ? 1 : 2);
11771       d.perm[i] = ei;
11772       perm[i] = ei;
11773     }
11774
11775   switch (which)
11776     {
11777     default:
11778       gcc_unreachable();
11779
11780     case 3:
11781       if (d.testing_p || !rtx_equal_p (d.op0, d.op1))
11782         {
11783           d.one_operand_p = false;
11784           break;
11785         }
11786
11787       /* The elements of PERM do not suggest that only the first operand
11788          is used, but both operands are identical.  Allow easier matching
11789          of the permutation by folding the permutation into the single
11790          input vector.  */
11791       for (i = 0; i < nelt; ++i)
11792         if (d.perm[i] >= nelt)
11793           d.perm[i] -= nelt;
11794       /* FALLTHRU */
11795
11796     case 1:
11797       d.op1 = d.op0;
11798       d.one_operand_p = true;
11799       break;
11800
11801     case 2:
11802       for (i = 0; i < nelt; ++i)
11803         d.perm[i] -= nelt;
11804       d.op0 = d.op1;
11805       d.one_operand_p = true;
11806       break;
11807     }
11808
11809   if (d.testing_p)
11810     {
11811       /* We have to go through the motions and see if we can
11812          figure out how to generate the requested permutation.  */
11813       d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
11814       d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
11815       if (!d.one_operand_p)
11816         d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
11817
11818       start_sequence ();
11819       bool ret = ia64_expand_vec_perm_const_1 (&d);
11820       end_sequence ();
11821
11822       return ret;
11823     }
11824
11825   if (ia64_expand_vec_perm_const_1 (&d))
11826     return true;
11827
11828   /* If the mask says both arguments are needed, but they are the same,
11829      the above tried to expand with one_operand_p true.  If that didn't
11830      work, retry with one_operand_p false, as that's what we used in _ok.  */
11831   if (which == 3 && d.one_operand_p)
11832     {
11833       memcpy (d.perm, perm, sizeof (perm));
11834       d.one_operand_p = false;
11835       return ia64_expand_vec_perm_const_1 (&d);
11836     }
11837
11838   return false;
11839 }
11840
11841 void
11842 ia64_expand_vec_setv2sf (rtx operands[3])
11843 {
11844   struct expand_vec_perm_d d;
11845   unsigned int which;
11846   bool ok;
11847
11848   d.target = operands[0];
11849   d.op0 = operands[0];
11850   d.op1 = gen_reg_rtx (V2SFmode);
11851   d.vmode = V2SFmode;
11852   d.nelt = 2;
11853   d.one_operand_p = false;
11854   d.testing_p = false;
11855
11856   which = INTVAL (operands[2]);
11857   gcc_assert (which <= 1);
11858   d.perm[0] = 1 - which;
11859   d.perm[1] = which + 2;
11860
11861   emit_insn (gen_fpack (d.op1, operands[1], CONST0_RTX (SFmode)));
11862
11863   ok = ia64_expand_vec_perm_const_1 (&d);
11864   gcc_assert (ok);
11865 }
11866
11867 void
11868 ia64_expand_vec_perm_even_odd (rtx target, rtx op0, rtx op1, int odd)
11869 {
11870   struct expand_vec_perm_d d;
11871   machine_mode vmode = GET_MODE (target);
11872   unsigned int i, nelt = GET_MODE_NUNITS (vmode);
11873   bool ok;
11874
11875   d.target = target;
11876   d.op0 = op0;
11877   d.op1 = op1;
11878   d.vmode = vmode;
11879   d.nelt = nelt;
11880   d.one_operand_p = false;
11881   d.testing_p = false;
11882
11883   for (i = 0; i < nelt; ++i)
11884     d.perm[i] = i * 2 + odd;
11885
11886   ok = ia64_expand_vec_perm_const_1 (&d);
11887   gcc_assert (ok);
11888 }
11889
11890 /* Implement TARGET_CAN_CHANGE_MODE_CLASS.
11891
11892    In BR regs, we can't change the DImode at all.
11893    In FP regs, we can't change FP values to integer values and vice versa,
11894    but we can change e.g. DImode to SImode, and V2SFmode into DImode.  */
11895
11896 static bool
11897 ia64_can_change_mode_class (machine_mode from, machine_mode to,
11898                             reg_class_t rclass)
11899 {
11900   if (reg_classes_intersect_p (rclass, BR_REGS))
11901     return from == to;
11902   if (SCALAR_FLOAT_MODE_P (from) != SCALAR_FLOAT_MODE_P (to))
11903     return !reg_classes_intersect_p (rclass, FR_REGS);
11904   return true;
11905 }
11906
11907 #include "gt-ia64.h"