gcc/config/ia64/ia64.c

   1 /* Definitions of target machine for GNU compiler.
   2    Copyright (C) 1999-2014 Free Software Foundation, Inc.
   3    Contributed by James E. Wilson <wilson@cygnus.com> and
   4                   David Mosberger <davidm@hpl.hp.com>.
   5
   6 This file is part of GCC.
   7
   8 GCC is free software; you can redistribute it and/or modify
   9 it under the terms of the GNU General Public License as published by
  10 the Free Software Foundation; either version 3, or (at your option)
  11 any later version.
  12
  13 GCC is distributed in the hope that it will be useful,
  14 but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 GNU General Public License for more details.
  17
  18 You should have received a copy of the GNU General Public License
  19 along with GCC; see the file COPYING3.  If not see
  20 <http://www.gnu.org/licenses/>.  */
  21
  22 #include "config.h"
  23 #include "system.h"
  24 #include "coretypes.h"
  25 #include "tm.h"
  26 #include "rtl.h"
  27 #include "tree.h"
  28 #include "stringpool.h"
  29 #include "stor-layout.h"
  30 #include "calls.h"
  31 #include "varasm.h"
  32 #include "regs.h"
  33 #include "hard-reg-set.h"
  34 #include "insn-config.h"
  35 #include "conditions.h"
  36 #include "output.h"
  37 #include "insn-attr.h"
  38 #include "flags.h"
  39 #include "recog.h"
  40 #include "expr.h"
  41 #include "insn-codes.h"
  42 #include "optabs.h"
  43 #include "except.h"
  44 #include "hashtab.h"
  45 #include "hash-set.h"
  46 #include "vec.h"
  47 #include "machmode.h"
  48 #include "input.h"
  49 #include "function.h"
  50 #include "ggc.h"
  51 #include "predict.h"
  52 #include "dominance.h"
  53 #include "cfg.h"
  54 #include "cfgrtl.h"
  55 #include "cfganal.h"
  56 #include "lcm.h"
  57 #include "cfgbuild.h"
  58 #include "cfgcleanup.h"
  59 #include "basic-block.h"
  60 #include "libfuncs.h"
  61 #include "diagnostic-core.h"
  62 #include "sched-int.h"
  63 #include "timevar.h"
  64 #include "target.h"
  65 #include "target-def.h"
  66 #include "common/common-target.h"
  67 #include "tm_p.h"
  68 #include "hash-table.h"
  69 #include "langhooks.h"
  70 #include "tree-ssa-alias.h"
  71 #include "internal-fn.h"
  72 #include "gimple-fold.h"
  73 #include "tree-eh.h"
  74 #include "gimple-expr.h"
  75 #include "is-a.h"
  76 #include "gimple.h"
  77 #include "gimplify.h"
  78 #include "intl.h"
  79 #include "df.h"
  80 #include "debug.h"
  81 #include "params.h"
  82 #include "dbgcnt.h"
  83 #include "tm-constrs.h"
  84 #include "sel-sched.h"
  85 #include "reload.h"
  86 #include "opts.h"
  87 #include "dumpfile.h"
  88 #include "builtins.h"
  89
  90 /* This is used for communication between ASM_OUTPUT_LABEL and
  91    ASM_OUTPUT_LABELREF.  */
  92 int ia64_asm_output_label = 0;
  93
  94 /* Register names for ia64_expand_prologue.  */
  95 static const char * const ia64_reg_numbers[96] =
  96 { "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
  97   "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
  98   "r48", "r49", "r50", "r51", "r52", "r53", "r54", "r55",
  99   "r56", "r57", "r58", "r59", "r60", "r61", "r62", "r63",
 100   "r64", "r65", "r66", "r67", "r68", "r69", "r70", "r71",
 101   "r72", "r73", "r74", "r75", "r76", "r77", "r78", "r79",
 102   "r80", "r81", "r82", "r83", "r84", "r85", "r86", "r87",
 103   "r88", "r89", "r90", "r91", "r92", "r93", "r94", "r95",
 104   "r96", "r97", "r98", "r99", "r100","r101","r102","r103",
 105   "r104","r105","r106","r107","r108","r109","r110","r111",
 106   "r112","r113","r114","r115","r116","r117","r118","r119",
 107   "r120","r121","r122","r123","r124","r125","r126","r127"};
 108
 109 /* ??? These strings could be shared with REGISTER_NAMES.  */
 110 static const char * const ia64_input_reg_names[8] =
 111 { "in0",  "in1",  "in2",  "in3",  "in4",  "in5",  "in6",  "in7" };
 112
 113 /* ??? These strings could be shared with REGISTER_NAMES.  */
 114 static const char * const ia64_local_reg_names[80] =
 115 { "loc0", "loc1", "loc2", "loc3", "loc4", "loc5", "loc6", "loc7",
 116   "loc8", "loc9", "loc10","loc11","loc12","loc13","loc14","loc15",
 117   "loc16","loc17","loc18","loc19","loc20","loc21","loc22","loc23",
 118   "loc24","loc25","loc26","loc27","loc28","loc29","loc30","loc31",
 119   "loc32","loc33","loc34","loc35","loc36","loc37","loc38","loc39",
 120   "loc40","loc41","loc42","loc43","loc44","loc45","loc46","loc47",
 121   "loc48","loc49","loc50","loc51","loc52","loc53","loc54","loc55",
 122   "loc56","loc57","loc58","loc59","loc60","loc61","loc62","loc63",
 123   "loc64","loc65","loc66","loc67","loc68","loc69","loc70","loc71",
 124   "loc72","loc73","loc74","loc75","loc76","loc77","loc78","loc79" };
 125
 126 /* ??? These strings could be shared with REGISTER_NAMES.  */
 127 static const char * const ia64_output_reg_names[8] =
 128 { "out0", "out1", "out2", "out3", "out4", "out5", "out6", "out7" };
 129
 130 /* Variables which are this size or smaller are put in the sdata/sbss
 131    sections.  */
 132
 133 unsigned int ia64_section_threshold;
 134
 135 /* The following variable is used by the DFA insn scheduler.  The value is
 136    TRUE if we do insn bundling instead of insn scheduling.  */
 137 int bundling_p = 0;
 138
 139 enum ia64_frame_regs
 140 {
 141    reg_fp,
 142    reg_save_b0,
 143    reg_save_pr,
 144    reg_save_ar_pfs,
 145    reg_save_ar_unat,
 146    reg_save_ar_lc,
 147    reg_save_gp,
 148    number_of_ia64_frame_regs
 149 };
 150
 151 /* Structure to be filled in by ia64_compute_frame_size with register
 152    save masks and offsets for the current function.  */
 153
 154 struct ia64_frame_info
 155 {
 156   HOST_WIDE_INT total_size;     /* size of the stack frame, not including
 157                                    the caller's scratch area.  */
 158   HOST_WIDE_INT spill_cfa_off;  /* top of the reg spill area from the cfa.  */
 159   HOST_WIDE_INT spill_size;     /* size of the gr/br/fr spill area.  */
 160   HOST_WIDE_INT extra_spill_size;  /* size of spill area for others.  */
 161   HARD_REG_SET mask;            /* mask of saved registers.  */
 162   unsigned int gr_used_mask;    /* mask of registers in use as gr spill
 163                                    registers or long-term scratches.  */
 164   int n_spilled;                /* number of spilled registers.  */
 165   int r[number_of_ia64_frame_regs];  /* Frame related registers.  */
 166   int n_input_regs;             /* number of input registers used.  */
 167   int n_local_regs;             /* number of local registers used.  */
 168   int n_output_regs;            /* number of output registers used.  */
 169   int n_rotate_regs;            /* number of rotating registers used.  */
 170
 171   char need_regstk;             /* true if a .regstk directive needed.  */
 172   char initialized;             /* true if the data is finalized.  */
 173 };
 174
 175 /* Current frame information calculated by ia64_compute_frame_size.  */
 176 static struct ia64_frame_info current_frame_info;
 177 /* The actual registers that are emitted.  */
 178 static int emitted_frame_related_regs[number_of_ia64_frame_regs];
 179 \f
 180 static int ia64_first_cycle_multipass_dfa_lookahead (void);
 181 static void ia64_dependencies_evaluation_hook (rtx_insn *, rtx_insn *);
 182 static void ia64_init_dfa_pre_cycle_insn (void);
 183 static rtx ia64_dfa_pre_cycle_insn (void);
 184 static int ia64_first_cycle_multipass_dfa_lookahead_guard (rtx_insn *, int);
 185 static int ia64_dfa_new_cycle (FILE *, int, rtx_insn *, int, int, int *);
 186 static void ia64_h_i_d_extended (void);
 187 static void * ia64_alloc_sched_context (void);
 188 static void ia64_init_sched_context (void *, bool);
 189 static void ia64_set_sched_context (void *);
 190 static void ia64_clear_sched_context (void *);
 191 static void ia64_free_sched_context (void *);
 192 static int ia64_mode_to_int (machine_mode);
 193 static void ia64_set_sched_flags (spec_info_t);
 194 static ds_t ia64_get_insn_spec_ds (rtx_insn *);
 195 static ds_t ia64_get_insn_checked_ds (rtx_insn *);
 196 static bool ia64_skip_rtx_p (const_rtx);
 197 static int ia64_speculate_insn (rtx_insn *, ds_t, rtx *);
 198 static bool ia64_needs_block_p (ds_t);
 199 static rtx ia64_gen_spec_check (rtx_insn *, rtx_insn *, ds_t);
 200 static int ia64_spec_check_p (rtx);
 201 static int ia64_spec_check_src_p (rtx);
 202 static rtx gen_tls_get_addr (void);
 203 static rtx gen_thread_pointer (void);
 204 static int find_gr_spill (enum ia64_frame_regs, int);
 205 static int next_scratch_gr_reg (void);
 206 static void mark_reg_gr_used_mask (rtx, void *);
 207 static void ia64_compute_frame_size (HOST_WIDE_INT);
 208 static void setup_spill_pointers (int, rtx, HOST_WIDE_INT);
 209 static void finish_spill_pointers (void);
 210 static rtx spill_restore_mem (rtx, HOST_WIDE_INT);
 211 static void do_spill (rtx (*)(rtx, rtx, rtx), rtx, HOST_WIDE_INT, rtx);
 212 static void do_restore (rtx (*)(rtx, rtx, rtx), rtx, HOST_WIDE_INT);
 213 static rtx gen_movdi_x (rtx, rtx, rtx);
 214 static rtx gen_fr_spill_x (rtx, rtx, rtx);
 215 static rtx gen_fr_restore_x (rtx, rtx, rtx);
 216
 217 static void ia64_option_override (void);
 218 static bool ia64_can_eliminate (const int, const int);
 219 static machine_mode hfa_element_mode (const_tree, bool);
 220 static void ia64_setup_incoming_varargs (cumulative_args_t, machine_mode,
 221                                          tree, int *, int);
 222 static int ia64_arg_partial_bytes (cumulative_args_t, machine_mode,
 223                                    tree, bool);
 224 static rtx ia64_function_arg_1 (cumulative_args_t, machine_mode,
 225                                 const_tree, bool, bool);
 226 static rtx ia64_function_arg (cumulative_args_t, machine_mode,
 227                               const_tree, bool);
 228 static rtx ia64_function_incoming_arg (cumulative_args_t,
 229                                        machine_mode, const_tree, bool);
 230 static void ia64_function_arg_advance (cumulative_args_t, machine_mode,
 231                                        const_tree, bool);
 232 static unsigned int ia64_function_arg_boundary (machine_mode,
 233                                                 const_tree);
 234 static bool ia64_function_ok_for_sibcall (tree, tree);
 235 static bool ia64_return_in_memory (const_tree, const_tree);
 236 static rtx ia64_function_value (const_tree, const_tree, bool);
 237 static rtx ia64_libcall_value (machine_mode, const_rtx);
 238 static bool ia64_function_value_regno_p (const unsigned int);
 239 static int ia64_register_move_cost (machine_mode, reg_class_t,
 240                                     reg_class_t);
 241 static int ia64_memory_move_cost (machine_mode mode, reg_class_t,
 242                                   bool);
 243 static bool ia64_rtx_costs (rtx, int, int, int, int *, bool);
 244 static int ia64_unspec_may_trap_p (const_rtx, unsigned);
 245 static void fix_range (const char *);
 246 static struct machine_function * ia64_init_machine_status (void);
 247 static void emit_insn_group_barriers (FILE *);
 248 static void emit_all_insn_group_barriers (FILE *);
 249 static void final_emit_insn_group_barriers (FILE *);
 250 static void emit_predicate_relation_info (void);
 251 static void ia64_reorg (void);
 252 static bool ia64_in_small_data_p (const_tree);
 253 static void process_epilogue (FILE *, rtx, bool, bool);
 254
 255 static bool ia64_assemble_integer (rtx, unsigned int, int);
 256 static void ia64_output_function_prologue (FILE *, HOST_WIDE_INT);
 257 static void ia64_output_function_epilogue (FILE *, HOST_WIDE_INT);
 258 static void ia64_output_function_end_prologue (FILE *);
 259
 260 static void ia64_print_operand (FILE *, rtx, int);
 261 static void ia64_print_operand_address (FILE *, rtx);
 262 static bool ia64_print_operand_punct_valid_p (unsigned char code);
 263
 264 static int ia64_issue_rate (void);
 265 static int ia64_adjust_cost_2 (rtx_insn *, int, rtx_insn *, int, dw_t);
 266 static void ia64_sched_init (FILE *, int, int);
 267 static void ia64_sched_init_global (FILE *, int, int);
 268 static void ia64_sched_finish_global (FILE *, int);
 269 static void ia64_sched_finish (FILE *, int);
 270 static int ia64_dfa_sched_reorder (FILE *, int, rtx_insn **, int *, int, int);
 271 static int ia64_sched_reorder (FILE *, int, rtx_insn **, int *, int);
 272 static int ia64_sched_reorder2 (FILE *, int, rtx_insn **, int *, int);
 273 static int ia64_variable_issue (FILE *, int, rtx_insn *, int);
 274
 275 static void ia64_asm_unwind_emit (FILE *, rtx_insn *);
 276 static void ia64_asm_emit_except_personality (rtx);
 277 static void ia64_asm_init_sections (void);
 278
 279 static enum unwind_info_type ia64_debug_unwind_info (void);
 280
 281 static struct bundle_state *get_free_bundle_state (void);
 282 static void free_bundle_state (struct bundle_state *);
 283 static void initiate_bundle_states (void);
 284 static void finish_bundle_states (void);
 285 static int insert_bundle_state (struct bundle_state *);
 286 static void initiate_bundle_state_table (void);
 287 static void finish_bundle_state_table (void);
 288 static int try_issue_nops (struct bundle_state *, int);
 289 static int try_issue_insn (struct bundle_state *, rtx);
 290 static void issue_nops_and_insn (struct bundle_state *, int, rtx_insn *,
 291                                  int, int);
 292 static int get_max_pos (state_t);
 293 static int get_template (state_t, int);
 294
 295 static rtx_insn *get_next_important_insn (rtx_insn *, rtx_insn *);
 296 static bool important_for_bundling_p (rtx_insn *);
 297 static bool unknown_for_bundling_p (rtx_insn *);
 298 static void bundling (FILE *, int, rtx_insn *, rtx_insn *);
 299
 300 static void ia64_output_mi_thunk (FILE *, tree, HOST_WIDE_INT,
 301                                   HOST_WIDE_INT, tree);
 302 static void ia64_file_start (void);
 303 static void ia64_globalize_decl_name (FILE *, tree);
 304
 305 static int ia64_hpux_reloc_rw_mask (void) ATTRIBUTE_UNUSED;
 306 static int ia64_reloc_rw_mask (void) ATTRIBUTE_UNUSED;
 307 static section *ia64_select_rtx_section (machine_mode, rtx,
 308                                          unsigned HOST_WIDE_INT);
 309 static void ia64_output_dwarf_dtprel (FILE *, int, rtx)
 310      ATTRIBUTE_UNUSED;
 311 static unsigned int ia64_section_type_flags (tree, const char *, int);
 312 static void ia64_init_libfuncs (void)
 313      ATTRIBUTE_UNUSED;
 314 static void ia64_hpux_init_libfuncs (void)
 315      ATTRIBUTE_UNUSED;
 316 static void ia64_sysv4_init_libfuncs (void)
 317      ATTRIBUTE_UNUSED;
 318 static void ia64_vms_init_libfuncs (void)
 319      ATTRIBUTE_UNUSED;
 320 static void ia64_soft_fp_init_libfuncs (void)
 321      ATTRIBUTE_UNUSED;
 322 static bool ia64_vms_valid_pointer_mode (machine_mode mode)
 323      ATTRIBUTE_UNUSED;
 324 static tree ia64_vms_common_object_attribute (tree *, tree, tree, int, bool *)
 325      ATTRIBUTE_UNUSED;
 326
 327 static tree ia64_handle_model_attribute (tree *, tree, tree, int, bool *);
 328 static tree ia64_handle_version_id_attribute (tree *, tree, tree, int, bool *);
 329 static void ia64_encode_section_info (tree, rtx, int);
 330 static rtx ia64_struct_value_rtx (tree, int);
 331 static tree ia64_gimplify_va_arg (tree, tree, gimple_seq *, gimple_seq *);
 332 static bool ia64_scalar_mode_supported_p (machine_mode mode);
 333 static bool ia64_vector_mode_supported_p (machine_mode mode);
 334 static bool ia64_libgcc_floating_mode_supported_p (machine_mode mode);
 335 static bool ia64_legitimate_constant_p (machine_mode, rtx);
 336 static bool ia64_legitimate_address_p (machine_mode, rtx, bool);
 337 static bool ia64_cannot_force_const_mem (machine_mode, rtx);
 338 static const char *ia64_mangle_type (const_tree);
 339 static const char *ia64_invalid_conversion (const_tree, const_tree);
 340 static const char *ia64_invalid_unary_op (int, const_tree);
 341 static const char *ia64_invalid_binary_op (int, const_tree, const_tree);
 342 static machine_mode ia64_c_mode_for_suffix (char);
 343 static void ia64_trampoline_init (rtx, tree, rtx);
 344 static void ia64_override_options_after_change (void);
 345 static bool ia64_member_type_forces_blk (const_tree, machine_mode);
 346
 347 static tree ia64_builtin_decl (unsigned, bool);
 348
 349 static reg_class_t ia64_preferred_reload_class (rtx, reg_class_t);
 350 static machine_mode ia64_get_reg_raw_mode (int regno);
 351 static section * ia64_hpux_function_section (tree, enum node_frequency,
 352                                              bool, bool);
 353
 354 static bool ia64_vectorize_vec_perm_const_ok (machine_mode vmode,
 355                                               const unsigned char *sel);
 356
 357 #define MAX_VECT_LEN    8
 358
 359 struct expand_vec_perm_d
 360 {
 361   rtx target, op0, op1;
 362   unsigned char perm[MAX_VECT_LEN];
 363   machine_mode vmode;
 364   unsigned char nelt;
 365   bool one_operand_p;
 366   bool testing_p;
 367 };
 368
 369 static bool ia64_expand_vec_perm_const_1 (struct expand_vec_perm_d *d);
 370
 371 \f
 372 /* Table of valid machine attributes.  */
 373 static const struct attribute_spec ia64_attribute_table[] =
 374 {
 375   /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
 376        affects_type_identity } */
 377   { "syscall_linkage", 0, 0, false, true,  true,  NULL, false },
 378   { "model",           1, 1, true, false, false, ia64_handle_model_attribute,
 379     false },
 380 #if TARGET_ABI_OPEN_VMS
 381   { "common_object",   1, 1, true, false, false,
 382     ia64_vms_common_object_attribute, false },
 383 #endif
 384   { "version_id",      1, 1, true, false, false,
 385     ia64_handle_version_id_attribute, false },
 386   { NULL,              0, 0, false, false, false, NULL, false }
 387 };
 388
 389 /* Initialize the GCC target structure.  */
 390 #undef TARGET_ATTRIBUTE_TABLE
 391 #define TARGET_ATTRIBUTE_TABLE ia64_attribute_table
 392
 393 #undef TARGET_INIT_BUILTINS
 394 #define TARGET_INIT_BUILTINS ia64_init_builtins
 395
 396 #undef TARGET_EXPAND_BUILTIN
 397 #define TARGET_EXPAND_BUILTIN ia64_expand_builtin
 398
 399 #undef TARGET_BUILTIN_DECL
 400 #define TARGET_BUILTIN_DECL ia64_builtin_decl
 401
 402 #undef TARGET_ASM_BYTE_OP
 403 #define TARGET_ASM_BYTE_OP "\tdata1\t"
 404 #undef TARGET_ASM_ALIGNED_HI_OP
 405 #define TARGET_ASM_ALIGNED_HI_OP "\tdata2\t"
 406 #undef TARGET_ASM_ALIGNED_SI_OP
 407 #define TARGET_ASM_ALIGNED_SI_OP "\tdata4\t"
 408 #undef TARGET_ASM_ALIGNED_DI_OP
 409 #define TARGET_ASM_ALIGNED_DI_OP "\tdata8\t"
 410 #undef TARGET_ASM_UNALIGNED_HI_OP
 411 #define TARGET_ASM_UNALIGNED_HI_OP "\tdata2.ua\t"
 412 #undef TARGET_ASM_UNALIGNED_SI_OP
 413 #define TARGET_ASM_UNALIGNED_SI_OP "\tdata4.ua\t"
 414 #undef TARGET_ASM_UNALIGNED_DI_OP
 415 #define TARGET_ASM_UNALIGNED_DI_OP "\tdata8.ua\t"
 416 #undef TARGET_ASM_INTEGER
 417 #define TARGET_ASM_INTEGER ia64_assemble_integer
 418
 419 #undef TARGET_OPTION_OVERRIDE
 420 #define TARGET_OPTION_OVERRIDE ia64_option_override
 421
 422 #undef TARGET_ASM_FUNCTION_PROLOGUE
 423 #define TARGET_ASM_FUNCTION_PROLOGUE ia64_output_function_prologue
 424 #undef TARGET_ASM_FUNCTION_END_PROLOGUE
 425 #define TARGET_ASM_FUNCTION_END_PROLOGUE ia64_output_function_end_prologue
 426 #undef TARGET_ASM_FUNCTION_EPILOGUE
 427 #define TARGET_ASM_FUNCTION_EPILOGUE ia64_output_function_epilogue
 428
 429 #undef TARGET_PRINT_OPERAND
 430 #define TARGET_PRINT_OPERAND ia64_print_operand
 431 #undef TARGET_PRINT_OPERAND_ADDRESS
 432 #define TARGET_PRINT_OPERAND_ADDRESS ia64_print_operand_address
 433 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
 434 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ia64_print_operand_punct_valid_p
 435
 436 #undef TARGET_IN_SMALL_DATA_P
 437 #define TARGET_IN_SMALL_DATA_P  ia64_in_small_data_p
 438
 439 #undef TARGET_SCHED_ADJUST_COST_2
 440 #define TARGET_SCHED_ADJUST_COST_2 ia64_adjust_cost_2
 441 #undef TARGET_SCHED_ISSUE_RATE
 442 #define TARGET_SCHED_ISSUE_RATE ia64_issue_rate
 443 #undef TARGET_SCHED_VARIABLE_ISSUE
 444 #define TARGET_SCHED_VARIABLE_ISSUE ia64_variable_issue
 445 #undef TARGET_SCHED_INIT
 446 #define TARGET_SCHED_INIT ia64_sched_init
 447 #undef TARGET_SCHED_FINISH
 448 #define TARGET_SCHED_FINISH ia64_sched_finish
 449 #undef TARGET_SCHED_INIT_GLOBAL
 450 #define TARGET_SCHED_INIT_GLOBAL ia64_sched_init_global
 451 #undef TARGET_SCHED_FINISH_GLOBAL
 452 #define TARGET_SCHED_FINISH_GLOBAL ia64_sched_finish_global
 453 #undef TARGET_SCHED_REORDER
 454 #define TARGET_SCHED_REORDER ia64_sched_reorder
 455 #undef TARGET_SCHED_REORDER2
 456 #define TARGET_SCHED_REORDER2 ia64_sched_reorder2
 457
 458 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
 459 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK ia64_dependencies_evaluation_hook
 460
 461 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
 462 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD ia64_first_cycle_multipass_dfa_lookahead
 463
 464 #undef TARGET_SCHED_INIT_DFA_PRE_CYCLE_INSN
 465 #define TARGET_SCHED_INIT_DFA_PRE_CYCLE_INSN ia64_init_dfa_pre_cycle_insn
 466 #undef TARGET_SCHED_DFA_PRE_CYCLE_INSN
 467 #define TARGET_SCHED_DFA_PRE_CYCLE_INSN ia64_dfa_pre_cycle_insn
 468
 469 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
 470 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD\
 471   ia64_first_cycle_multipass_dfa_lookahead_guard
 472
 473 #undef TARGET_SCHED_DFA_NEW_CYCLE
 474 #define TARGET_SCHED_DFA_NEW_CYCLE ia64_dfa_new_cycle
 475
 476 #undef TARGET_SCHED_H_I_D_EXTENDED
 477 #define TARGET_SCHED_H_I_D_EXTENDED ia64_h_i_d_extended
 478
 479 #undef TARGET_SCHED_ALLOC_SCHED_CONTEXT
 480 #define TARGET_SCHED_ALLOC_SCHED_CONTEXT ia64_alloc_sched_context
 481
 482 #undef TARGET_SCHED_INIT_SCHED_CONTEXT
 483 #define TARGET_SCHED_INIT_SCHED_CONTEXT ia64_init_sched_context
 484
 485 #undef TARGET_SCHED_SET_SCHED_CONTEXT
 486 #define TARGET_SCHED_SET_SCHED_CONTEXT ia64_set_sched_context
 487
 488 #undef TARGET_SCHED_CLEAR_SCHED_CONTEXT
 489 #define TARGET_SCHED_CLEAR_SCHED_CONTEXT ia64_clear_sched_context
 490
 491 #undef TARGET_SCHED_FREE_SCHED_CONTEXT
 492 #define TARGET_SCHED_FREE_SCHED_CONTEXT ia64_free_sched_context
 493
 494 #undef TARGET_SCHED_SET_SCHED_FLAGS
 495 #define TARGET_SCHED_SET_SCHED_FLAGS ia64_set_sched_flags
 496
 497 #undef TARGET_SCHED_GET_INSN_SPEC_DS
 498 #define TARGET_SCHED_GET_INSN_SPEC_DS ia64_get_insn_spec_ds
 499
 500 #undef TARGET_SCHED_GET_INSN_CHECKED_DS
 501 #define TARGET_SCHED_GET_INSN_CHECKED_DS ia64_get_insn_checked_ds
 502
 503 #undef TARGET_SCHED_SPECULATE_INSN
 504 #define TARGET_SCHED_SPECULATE_INSN ia64_speculate_insn
 505
 506 #undef TARGET_SCHED_NEEDS_BLOCK_P
 507 #define TARGET_SCHED_NEEDS_BLOCK_P ia64_needs_block_p
 508
 509 #undef TARGET_SCHED_GEN_SPEC_CHECK
 510 #define TARGET_SCHED_GEN_SPEC_CHECK ia64_gen_spec_check
 511
 512 #undef TARGET_SCHED_SKIP_RTX_P
 513 #define TARGET_SCHED_SKIP_RTX_P ia64_skip_rtx_p
 514
 515 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
 516 #define TARGET_FUNCTION_OK_FOR_SIBCALL ia64_function_ok_for_sibcall
 517 #undef TARGET_ARG_PARTIAL_BYTES
 518 #define TARGET_ARG_PARTIAL_BYTES ia64_arg_partial_bytes
 519 #undef TARGET_FUNCTION_ARG
 520 #define TARGET_FUNCTION_ARG ia64_function_arg
 521 #undef TARGET_FUNCTION_INCOMING_ARG
 522 #define TARGET_FUNCTION_INCOMING_ARG ia64_function_incoming_arg
 523 #undef TARGET_FUNCTION_ARG_ADVANCE
 524 #define TARGET_FUNCTION_ARG_ADVANCE ia64_function_arg_advance
 525 #undef TARGET_FUNCTION_ARG_BOUNDARY
 526 #define TARGET_FUNCTION_ARG_BOUNDARY ia64_function_arg_boundary
 527
 528 #undef TARGET_ASM_OUTPUT_MI_THUNK
 529 #define TARGET_ASM_OUTPUT_MI_THUNK ia64_output_mi_thunk
 530 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
 531 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK hook_bool_const_tree_hwi_hwi_const_tree_true
 532
 533 #undef TARGET_ASM_FILE_START
 534 #define TARGET_ASM_FILE_START ia64_file_start
 535
 536 #undef TARGET_ASM_GLOBALIZE_DECL_NAME
 537 #define TARGET_ASM_GLOBALIZE_DECL_NAME ia64_globalize_decl_name
 538
 539 #undef TARGET_REGISTER_MOVE_COST
 540 #define TARGET_REGISTER_MOVE_COST ia64_register_move_cost
 541 #undef TARGET_MEMORY_MOVE_COST
 542 #define TARGET_MEMORY_MOVE_COST ia64_memory_move_cost
 543 #undef TARGET_RTX_COSTS
 544 #define TARGET_RTX_COSTS ia64_rtx_costs
 545 #undef TARGET_ADDRESS_COST
 546 #define TARGET_ADDRESS_COST hook_int_rtx_mode_as_bool_0
 547
 548 #undef TARGET_UNSPEC_MAY_TRAP_P
 549 #define TARGET_UNSPEC_MAY_TRAP_P ia64_unspec_may_trap_p
 550
 551 #undef TARGET_MACHINE_DEPENDENT_REORG
 552 #define TARGET_MACHINE_DEPENDENT_REORG ia64_reorg
 553
 554 #undef TARGET_ENCODE_SECTION_INFO
 555 #define TARGET_ENCODE_SECTION_INFO ia64_encode_section_info
 556
 557 #undef  TARGET_SECTION_TYPE_FLAGS
 558 #define TARGET_SECTION_TYPE_FLAGS  ia64_section_type_flags
 559
 560 #ifdef HAVE_AS_TLS
 561 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
 562 #define TARGET_ASM_OUTPUT_DWARF_DTPREL ia64_output_dwarf_dtprel
 563 #endif
 564
 565 /* ??? Investigate.  */
 566 #if 0
 567 #undef TARGET_PROMOTE_PROTOTYPES
 568 #define TARGET_PROMOTE_PROTOTYPES hook_bool_tree_true
 569 #endif
 570
 571 #undef TARGET_FUNCTION_VALUE
 572 #define TARGET_FUNCTION_VALUE ia64_function_value
 573 #undef TARGET_LIBCALL_VALUE
 574 #define TARGET_LIBCALL_VALUE ia64_libcall_value
 575 #undef TARGET_FUNCTION_VALUE_REGNO_P
 576 #define TARGET_FUNCTION_VALUE_REGNO_P ia64_function_value_regno_p
 577
 578 #undef TARGET_STRUCT_VALUE_RTX
 579 #define TARGET_STRUCT_VALUE_RTX ia64_struct_value_rtx
 580 #undef TARGET_RETURN_IN_MEMORY
 581 #define TARGET_RETURN_IN_MEMORY ia64_return_in_memory
 582 #undef TARGET_SETUP_INCOMING_VARARGS
 583 #define TARGET_SETUP_INCOMING_VARARGS ia64_setup_incoming_varargs
 584 #undef TARGET_STRICT_ARGUMENT_NAMING
 585 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
 586 #undef TARGET_MUST_PASS_IN_STACK
 587 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
 588 #undef TARGET_GET_RAW_RESULT_MODE
 589 #define TARGET_GET_RAW_RESULT_MODE ia64_get_reg_raw_mode
 590 #undef TARGET_GET_RAW_ARG_MODE
 591 #define TARGET_GET_RAW_ARG_MODE ia64_get_reg_raw_mode
 592
 593 #undef TARGET_MEMBER_TYPE_FORCES_BLK
 594 #define TARGET_MEMBER_TYPE_FORCES_BLK ia64_member_type_forces_blk
 595
 596 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
 597 #define TARGET_GIMPLIFY_VA_ARG_EXPR ia64_gimplify_va_arg
 598
 599 #undef TARGET_ASM_UNWIND_EMIT
 600 #define TARGET_ASM_UNWIND_EMIT ia64_asm_unwind_emit
 601 #undef TARGET_ASM_EMIT_EXCEPT_PERSONALITY
 602 #define TARGET_ASM_EMIT_EXCEPT_PERSONALITY  ia64_asm_emit_except_personality
 603 #undef TARGET_ASM_INIT_SECTIONS
 604 #define TARGET_ASM_INIT_SECTIONS  ia64_asm_init_sections
 605
 606 #undef TARGET_DEBUG_UNWIND_INFO
 607 #define TARGET_DEBUG_UNWIND_INFO  ia64_debug_unwind_info
 608
 609 #undef TARGET_SCALAR_MODE_SUPPORTED_P
 610 #define TARGET_SCALAR_MODE_SUPPORTED_P ia64_scalar_mode_supported_p
 611 #undef TARGET_VECTOR_MODE_SUPPORTED_P
 612 #define TARGET_VECTOR_MODE_SUPPORTED_P ia64_vector_mode_supported_p
 613
 614 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
 615 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
 616   ia64_libgcc_floating_mode_supported_p
 617
 618 /* ia64 architecture manual 4.4.7: ... reads, writes, and flushes may occur
 619    in an order different from the specified program order.  */
 620 #undef TARGET_RELAXED_ORDERING
 621 #define TARGET_RELAXED_ORDERING true
 622
 623 #undef TARGET_LEGITIMATE_CONSTANT_P
 624 #define TARGET_LEGITIMATE_CONSTANT_P ia64_legitimate_constant_p
 625 #undef TARGET_LEGITIMATE_ADDRESS_P
 626 #define TARGET_LEGITIMATE_ADDRESS_P ia64_legitimate_address_p
 627
 628 #undef TARGET_CANNOT_FORCE_CONST_MEM
 629 #define TARGET_CANNOT_FORCE_CONST_MEM ia64_cannot_force_const_mem
 630
 631 #undef TARGET_MANGLE_TYPE
 632 #define TARGET_MANGLE_TYPE ia64_mangle_type
 633
 634 #undef TARGET_INVALID_CONVERSION
 635 #define TARGET_INVALID_CONVERSION ia64_invalid_conversion
 636 #undef TARGET_INVALID_UNARY_OP
 637 #define TARGET_INVALID_UNARY_OP ia64_invalid_unary_op
 638 #undef TARGET_INVALID_BINARY_OP
 639 #define TARGET_INVALID_BINARY_OP ia64_invalid_binary_op
 640
 641 #undef TARGET_C_MODE_FOR_SUFFIX
 642 #define TARGET_C_MODE_FOR_SUFFIX ia64_c_mode_for_suffix
 643
 644 #undef TARGET_CAN_ELIMINATE
 645 #define TARGET_CAN_ELIMINATE ia64_can_eliminate
 646
 647 #undef TARGET_TRAMPOLINE_INIT
 648 #define TARGET_TRAMPOLINE_INIT ia64_trampoline_init
 649
 650 #undef TARGET_CAN_USE_DOLOOP_P
 651 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
 652 #undef TARGET_INVALID_WITHIN_DOLOOP
 653 #define TARGET_INVALID_WITHIN_DOLOOP hook_constcharptr_const_rtx_insn_null
 654
 655 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
 656 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE ia64_override_options_after_change
 657
 658 #undef TARGET_PREFERRED_RELOAD_CLASS
 659 #define TARGET_PREFERRED_RELOAD_CLASS ia64_preferred_reload_class
 660
 661 #undef TARGET_DELAY_SCHED2
 662 #define TARGET_DELAY_SCHED2 true
 663
 664 /* Variable tracking should be run after all optimizations which
 665    change order of insns.  It also needs a valid CFG.  */
 666 #undef TARGET_DELAY_VARTRACK
 667 #define TARGET_DELAY_VARTRACK true
 668
 669 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
 670 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK ia64_vectorize_vec_perm_const_ok
 671
 672 struct gcc_target targetm = TARGET_INITIALIZER;
 673 \f
 674 typedef enum
 675   {
 676     ADDR_AREA_NORMAL,   /* normal address area */
 677     ADDR_AREA_SMALL     /* addressable by "addl" (-2MB < addr < 2MB) */
 678   }
 679 ia64_addr_area;
 680
 681 static GTY(()) tree small_ident1;
 682 static GTY(()) tree small_ident2;
 683
 684 static void
 685 init_idents (void)
 686 {
 687   if (small_ident1 == 0)
 688     {
 689       small_ident1 = get_identifier ("small");
 690       small_ident2 = get_identifier ("__small__");
 691     }
 692 }
 693
 694 /* Retrieve the address area that has been chosen for the given decl.  */
 695
 696 static ia64_addr_area
 697 ia64_get_addr_area (tree decl)
 698 {
 699   tree model_attr;
 700
 701   model_attr = lookup_attribute ("model", DECL_ATTRIBUTES (decl));
 702   if (model_attr)
 703     {
 704       tree id;
 705
 706       init_idents ();
 707       id = TREE_VALUE (TREE_VALUE (model_attr));
 708       if (id == small_ident1 || id == small_ident2)
 709         return ADDR_AREA_SMALL;
 710     }
 711   return ADDR_AREA_NORMAL;
 712 }
 713
 714 static tree
 715 ia64_handle_model_attribute (tree *node, tree name, tree args,
 716                              int flags ATTRIBUTE_UNUSED, bool *no_add_attrs)
 717 {
 718   ia64_addr_area addr_area = ADDR_AREA_NORMAL;
 719   ia64_addr_area area;
 720   tree arg, decl = *node;
 721
 722   init_idents ();
 723   arg = TREE_VALUE (args);
 724   if (arg == small_ident1 || arg == small_ident2)
 725     {
 726       addr_area = ADDR_AREA_SMALL;
 727     }
 728   else
 729     {
 730       warning (OPT_Wattributes, "invalid argument of %qE attribute",
 731                name);
 732       *no_add_attrs = true;
 733     }
 734
 735   switch (TREE_CODE (decl))
 736     {
 737     case VAR_DECL:
 738       if ((DECL_CONTEXT (decl) && TREE_CODE (DECL_CONTEXT (decl))
 739            == FUNCTION_DECL)
 740           && !TREE_STATIC (decl))
 741         {
 742           error_at (DECL_SOURCE_LOCATION (decl),
 743                     "an address area attribute cannot be specified for "
 744                     "local variables");
 745           *no_add_attrs = true;
 746         }
 747       area = ia64_get_addr_area (decl);
 748       if (area != ADDR_AREA_NORMAL && addr_area != area)
 749         {
 750           error ("address area of %q+D conflicts with previous "
 751                  "declaration", decl);
 752           *no_add_attrs = true;
 753         }
 754       break;
 755
 756     case FUNCTION_DECL:
 757       error_at (DECL_SOURCE_LOCATION (decl),
 758                 "address area attribute cannot be specified for "
 759                 "functions");
 760       *no_add_attrs = true;
 761       break;
 762
 763     default:
 764       warning (OPT_Wattributes, "%qE attribute ignored",
 765                name);
 766       *no_add_attrs = true;
 767       break;
 768     }
 769
 770   return NULL_TREE;
 771 }
 772
 773 /* Part of the low level implementation of DEC Ada pragma Common_Object which
 774    enables the shared use of variables stored in overlaid linker areas
 775    corresponding to the use of Fortran COMMON.  */
 776
 777 static tree
 778 ia64_vms_common_object_attribute (tree *node, tree name, tree args,
 779                                   int flags ATTRIBUTE_UNUSED,
 780                                   bool *no_add_attrs)
 781 {
 782     tree decl = *node;
 783     tree id;
 784
 785     gcc_assert (DECL_P (decl));
 786
 787     DECL_COMMON (decl) = 1;
 788     id = TREE_VALUE (args);
 789     if (TREE_CODE (id) != IDENTIFIER_NODE && TREE_CODE (id) != STRING_CST)
 790       {
 791         error ("%qE attribute requires a string constant argument", name);
 792         *no_add_attrs = true;
 793         return NULL_TREE;
 794       }
 795     return NULL_TREE;
 796 }
 797
 798 /* Part of the low level implementation of DEC Ada pragma Common_Object.  */
 799
 800 void
 801 ia64_vms_output_aligned_decl_common (FILE *file, tree decl, const char *name,
 802                                      unsigned HOST_WIDE_INT size,
 803                                      unsigned int align)
 804 {
 805   tree attr = DECL_ATTRIBUTES (decl);
 806
 807   if (attr)
 808     attr = lookup_attribute ("common_object", attr);
 809   if (attr)
 810     {
 811       tree id = TREE_VALUE (TREE_VALUE (attr));
 812       const char *name;
 813
 814       if (TREE_CODE (id) == IDENTIFIER_NODE)
 815         name = IDENTIFIER_POINTER (id);
 816       else if (TREE_CODE (id) == STRING_CST)
 817         name = TREE_STRING_POINTER (id);
 818       else
 819         abort ();
 820
 821       fprintf (file, "\t.vms_common\t\"%s\",", name);
 822     }
 823   else
 824     fprintf (file, "%s", COMMON_ASM_OP);
 825
 826   /*  Code from elfos.h.  */
 827   assemble_name (file, name);
 828   fprintf (file, ","HOST_WIDE_INT_PRINT_UNSIGNED",%u",
 829            size, align / BITS_PER_UNIT);
 830
 831   fputc ('\n', file);
 832 }
 833
 834 static void
 835 ia64_encode_addr_area (tree decl, rtx symbol)
 836 {
 837   int flags;
 838
 839   flags = SYMBOL_REF_FLAGS (symbol);
 840   switch (ia64_get_addr_area (decl))
 841     {
 842     case ADDR_AREA_NORMAL: break;
 843     case ADDR_AREA_SMALL: flags |= SYMBOL_FLAG_SMALL_ADDR; break;
 844     default: gcc_unreachable ();
 845     }
 846   SYMBOL_REF_FLAGS (symbol) = flags;
 847 }
 848
 849 static void
 850 ia64_encode_section_info (tree decl, rtx rtl, int first)
 851 {
 852   default_encode_section_info (decl, rtl, first);
 853
 854   /* Careful not to prod global register variables.  */
 855   if (TREE_CODE (decl) == VAR_DECL
 856       && GET_CODE (DECL_RTL (decl)) == MEM
 857       && GET_CODE (XEXP (DECL_RTL (decl), 0)) == SYMBOL_REF
 858       && (TREE_STATIC (decl) || DECL_EXTERNAL (decl)))
 859     ia64_encode_addr_area (decl, XEXP (rtl, 0));
 860 }
 861 \f
 862 /* Return 1 if the operands of a move are ok.  */
 863
 864 int
 865 ia64_move_ok (rtx dst, rtx src)
 866 {
 867   /* If we're under init_recog_no_volatile, we'll not be able to use
 868      memory_operand.  So check the code directly and don't worry about
 869      the validity of the underlying address, which should have been
 870      checked elsewhere anyway.  */
 871   if (GET_CODE (dst) != MEM)
 872     return 1;
 873   if (GET_CODE (src) == MEM)
 874     return 0;
 875   if (register_operand (src, VOIDmode))
 876     return 1;
 877
 878   /* Otherwise, this must be a constant, and that either 0 or 0.0 or 1.0.  */
 879   if (INTEGRAL_MODE_P (GET_MODE (dst)))
 880     return src == const0_rtx;
 881   else
 882     return satisfies_constraint_G (src);
 883 }
 884
 885 /* Return 1 if the operands are ok for a floating point load pair.  */
 886
 887 int
 888 ia64_load_pair_ok (rtx dst, rtx src)
 889 {
 890   /* ??? There is a thinko in the implementation of the "x" constraint and the
 891      FP_REGS class.  The constraint will also reject (reg f30:TI) so we must
 892      also return false for it.  */
 893   if (GET_CODE (dst) != REG
 894       || !(FP_REGNO_P (REGNO (dst)) && FP_REGNO_P (REGNO (dst) + 1)))
 895     return 0;
 896   if (GET_CODE (src) != MEM || MEM_VOLATILE_P (src))
 897     return 0;
 898   switch (GET_CODE (XEXP (src, 0)))
 899     {
 900     case REG:
 901     case POST_INC:
 902       break;
 903     case POST_DEC:
 904       return 0;
 905     case POST_MODIFY:
 906       {
 907         rtx adjust = XEXP (XEXP (XEXP (src, 0), 1), 1);
 908
 909         if (GET_CODE (adjust) != CONST_INT
 910             || INTVAL (adjust) != GET_MODE_SIZE (GET_MODE (src)))
 911           return 0;
 912       }
 913       break;
 914     default:
 915       abort ();
 916     }
 917   return 1;
 918 }
 919
 920 int
 921 addp4_optimize_ok (rtx op1, rtx op2)
 922 {
 923   return (basereg_operand (op1, GET_MODE(op1)) !=
 924           basereg_operand (op2, GET_MODE(op2)));
 925 }
 926
 927 /* Check if OP is a mask suitable for use with SHIFT in a dep.z instruction.
 928    Return the length of the field, or <= 0 on failure.  */
 929
 930 int
 931 ia64_depz_field_mask (rtx rop, rtx rshift)
 932 {
 933   unsigned HOST_WIDE_INT op = INTVAL (rop);
 934   unsigned HOST_WIDE_INT shift = INTVAL (rshift);
 935
 936   /* Get rid of the zero bits we're shifting in.  */
 937   op >>= shift;
 938
 939   /* We must now have a solid block of 1's at bit 0.  */
 940   return exact_log2 (op + 1);
 941 }
 942
 943 /* Return the TLS model to use for ADDR.  */
 944
 945 static enum tls_model
 946 tls_symbolic_operand_type (rtx addr)
 947 {
 948   enum tls_model tls_kind = TLS_MODEL_NONE;
 949
 950   if (GET_CODE (addr) == CONST)
 951     {
 952       if (GET_CODE (XEXP (addr, 0)) == PLUS
 953           && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF)
 954         tls_kind = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (addr, 0), 0));
 955     }
 956   else if (GET_CODE (addr) == SYMBOL_REF)
 957     tls_kind = SYMBOL_REF_TLS_MODEL (addr);
 958
 959   return tls_kind;
 960 }
 961
 962 /* Returns true if REG (assumed to be a `reg' RTX) is valid for use
 963    as a base register.  */
 964
 965 static inline bool
 966 ia64_reg_ok_for_base_p (const_rtx reg, bool strict)
 967 {
 968   if (strict
 969       && REGNO_OK_FOR_BASE_P (REGNO (reg)))
 970     return true;
 971   else if (!strict
 972            && (GENERAL_REGNO_P (REGNO (reg))
 973                || !HARD_REGISTER_P (reg)))
 974     return true;
 975   else
 976     return false;
 977 }
 978
 979 static bool
 980 ia64_legitimate_address_reg (const_rtx reg, bool strict)
 981 {
 982   if ((REG_P (reg) && ia64_reg_ok_for_base_p (reg, strict))
 983       || (GET_CODE (reg) == SUBREG && REG_P (XEXP (reg, 0))
 984           && ia64_reg_ok_for_base_p (XEXP (reg, 0), strict)))
 985     return true;
 986
 987   return false;
 988 }
 989
 990 static bool
 991 ia64_legitimate_address_disp (const_rtx reg, const_rtx disp, bool strict)
 992 {
 993   if (GET_CODE (disp) == PLUS
 994       && rtx_equal_p (reg, XEXP (disp, 0))
 995       && (ia64_legitimate_address_reg (XEXP (disp, 1), strict)
 996           || (CONST_INT_P (XEXP (disp, 1))
 997               && IN_RANGE (INTVAL (XEXP (disp, 1)), -256, 255))))
 998     return true;
 999
1000   return false;
1001 }
1002
1003 /* Implement TARGET_LEGITIMATE_ADDRESS_P.  */
1004
1005 static bool
1006 ia64_legitimate_address_p (machine_mode mode ATTRIBUTE_UNUSED,
1007                            rtx x, bool strict)
1008 {
1009   if (ia64_legitimate_address_reg (x, strict))
1010     return true;
1011   else if ((GET_CODE (x) == POST_INC || GET_CODE (x) == POST_DEC)
1012            && ia64_legitimate_address_reg (XEXP (x, 0), strict)
1013            && XEXP (x, 0) != arg_pointer_rtx)
1014     return true;
1015   else if (GET_CODE (x) == POST_MODIFY
1016            && ia64_legitimate_address_reg (XEXP (x, 0), strict)
1017            && XEXP (x, 0) != arg_pointer_rtx
1018            && ia64_legitimate_address_disp (XEXP (x, 0), XEXP (x, 1), strict))
1019     return true;
1020   else
1021     return false;
1022 }
1023
1024 /* Return true if X is a constant that is valid for some immediate
1025    field in an instruction.  */
1026
1027 static bool
1028 ia64_legitimate_constant_p (machine_mode mode, rtx x)
1029 {
1030   switch (GET_CODE (x))
1031     {
1032     case CONST_INT:
1033     case LABEL_REF:
1034       return true;
1035
1036     case CONST_DOUBLE:
1037       if (GET_MODE (x) == VOIDmode || mode == SFmode || mode == DFmode)
1038         return true;
1039       return satisfies_constraint_G (x);
1040
1041     case CONST:
1042     case SYMBOL_REF:
1043       /* ??? Short term workaround for PR 28490.  We must make the code here
1044          match the code in ia64_expand_move and move_operand, even though they
1045          are both technically wrong.  */
1046       if (tls_symbolic_operand_type (x) == 0)
1047         {
1048           HOST_WIDE_INT addend = 0;
1049           rtx op = x;
1050
1051           if (GET_CODE (op) == CONST
1052               && GET_CODE (XEXP (op, 0)) == PLUS
1053               && GET_CODE (XEXP (XEXP (op, 0), 1)) == CONST_INT)
1054             {
1055               addend = INTVAL (XEXP (XEXP (op, 0), 1));
1056               op = XEXP (XEXP (op, 0), 0);
1057             }
1058
1059           if (any_offset_symbol_operand (op, mode)
1060               || function_operand (op, mode))
1061             return true;
1062           if (aligned_offset_symbol_operand (op, mode))
1063             return (addend & 0x3fff) == 0;
1064           return false;
1065         }
1066       return false;
1067
1068     case CONST_VECTOR:
1069       if (mode == V2SFmode)
1070         return satisfies_constraint_Y (x);
1071
1072       return (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
1073               && GET_MODE_SIZE (mode) <= 8);
1074
1075     default:
1076       return false;
1077     }
1078 }
1079
1080 /* Don't allow TLS addresses to get spilled to memory.  */
1081
1082 static bool
1083 ia64_cannot_force_const_mem (machine_mode mode, rtx x)
1084 {
1085   if (mode == RFmode)
1086     return true;
1087   return tls_symbolic_operand_type (x) != 0;
1088 }
1089
1090 /* Expand a symbolic constant load.  */
1091
1092 bool
1093 ia64_expand_load_address (rtx dest, rtx src)
1094 {
1095   gcc_assert (GET_CODE (dest) == REG);
1096
1097   /* ILP32 mode still loads 64-bits of data from the GOT.  This avoids
1098      having to pointer-extend the value afterward.  Other forms of address
1099      computation below are also more natural to compute as 64-bit quantities.
1100      If we've been given an SImode destination register, change it.  */
1101   if (GET_MODE (dest) != Pmode)
1102     dest = gen_rtx_REG_offset (dest, Pmode, REGNO (dest),
1103                                byte_lowpart_offset (Pmode, GET_MODE (dest)));
1104
1105   if (TARGET_NO_PIC)
1106     return false;
1107   if (small_addr_symbolic_operand (src, VOIDmode))
1108     return false;
1109
1110   if (TARGET_AUTO_PIC)
1111     emit_insn (gen_load_gprel64 (dest, src));
1112   else if (GET_CODE (src) == SYMBOL_REF && SYMBOL_REF_FUNCTION_P (src))
1113     emit_insn (gen_load_fptr (dest, src));
1114   else if (sdata_symbolic_operand (src, VOIDmode))
1115     emit_insn (gen_load_gprel (dest, src));
1116   else
1117     {
1118       HOST_WIDE_INT addend = 0;
1119       rtx tmp;
1120
1121       /* We did split constant offsets in ia64_expand_move, and we did try
1122          to keep them split in move_operand, but we also allowed reload to
1123          rematerialize arbitrary constants rather than spill the value to
1124          the stack and reload it.  So we have to be prepared here to split
1125          them apart again.  */
1126       if (GET_CODE (src) == CONST)
1127         {
1128           HOST_WIDE_INT hi, lo;
1129
1130           hi = INTVAL (XEXP (XEXP (src, 0), 1));
1131           lo = ((hi & 0x3fff) ^ 0x2000) - 0x2000;
1132           hi = hi - lo;
1133
1134           if (lo != 0)
1135             {
1136               addend = lo;
1137               src = plus_constant (Pmode, XEXP (XEXP (src, 0), 0), hi);
1138             }
1139         }
1140
1141       tmp = gen_rtx_HIGH (Pmode, src);
1142       tmp = gen_rtx_PLUS (Pmode, tmp, pic_offset_table_rtx);
1143       emit_insn (gen_rtx_SET (VOIDmode, dest, tmp));
1144
1145       tmp = gen_rtx_LO_SUM (Pmode, gen_const_mem (Pmode, dest), src);
1146       emit_insn (gen_rtx_SET (VOIDmode, dest, tmp));
1147
1148       if (addend)
1149         {
1150           tmp = gen_rtx_PLUS (Pmode, dest, GEN_INT (addend));
1151           emit_insn (gen_rtx_SET (VOIDmode, dest, tmp));
1152         }
1153     }
1154
1155   return true;
1156 }
1157
1158 static GTY(()) rtx gen_tls_tga;
1159 static rtx
1160 gen_tls_get_addr (void)
1161 {
1162   if (!gen_tls_tga)
1163     gen_tls_tga = init_one_libfunc ("__tls_get_addr");
1164   return gen_tls_tga;
1165 }
1166
1167 static GTY(()) rtx thread_pointer_rtx;
1168 static rtx
1169 gen_thread_pointer (void)
1170 {
1171   if (!thread_pointer_rtx)
1172     thread_pointer_rtx = gen_rtx_REG (Pmode, 13);
1173   return thread_pointer_rtx;
1174 }
1175
1176 static rtx
1177 ia64_expand_tls_address (enum tls_model tls_kind, rtx op0, rtx op1,
1178                          rtx orig_op1, HOST_WIDE_INT addend)
1179 {
1180   rtx tga_op1, tga_op2, tga_ret, tga_eqv, tmp;
1181   rtx_insn *insns;
1182   rtx orig_op0 = op0;
1183   HOST_WIDE_INT addend_lo, addend_hi;
1184
1185   switch (tls_kind)
1186     {
1187     case TLS_MODEL_GLOBAL_DYNAMIC:
1188       start_sequence ();
1189
1190       tga_op1 = gen_reg_rtx (Pmode);
1191       emit_insn (gen_load_dtpmod (tga_op1, op1));
1192
1193       tga_op2 = gen_reg_rtx (Pmode);
1194       emit_insn (gen_load_dtprel (tga_op2, op1));
1195
1196       tga_ret = emit_library_call_value (gen_tls_get_addr (), NULL_RTX,
1197                                          LCT_CONST, Pmode, 2, tga_op1,
1198                                          Pmode, tga_op2, Pmode);
1199
1200       insns = get_insns ();
1201       end_sequence ();
1202
1203       if (GET_MODE (op0) != Pmode)
1204         op0 = tga_ret;
1205       emit_libcall_block (insns, op0, tga_ret, op1);
1206       break;
1207
1208     case TLS_MODEL_LOCAL_DYNAMIC:
1209       /* ??? This isn't the completely proper way to do local-dynamic
1210          If the call to __tls_get_addr is used only by a single symbol,
1211          then we should (somehow) move the dtprel to the second arg
1212          to avoid the extra add.  */
1213       start_sequence ();
1214
1215       tga_op1 = gen_reg_rtx (Pmode);
1216       emit_insn (gen_load_dtpmod (tga_op1, op1));
1217
1218       tga_op2 = const0_rtx;
1219
1220       tga_ret = emit_library_call_value (gen_tls_get_addr (), NULL_RTX,
1221                                          LCT_CONST, Pmode, 2, tga_op1,
1222                                          Pmode, tga_op2, Pmode);
1223
1224       insns = get_insns ();
1225       end_sequence ();
1226
1227       tga_eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
1228                                 UNSPEC_LD_BASE);
1229       tmp = gen_reg_rtx (Pmode);
1230       emit_libcall_block (insns, tmp, tga_ret, tga_eqv);
1231
1232       if (!register_operand (op0, Pmode))
1233         op0 = gen_reg_rtx (Pmode);
1234       if (TARGET_TLS64)
1235         {
1236           emit_insn (gen_load_dtprel (op0, op1));
1237           emit_insn (gen_adddi3 (op0, tmp, op0));
1238         }
1239       else
1240         emit_insn (gen_add_dtprel (op0, op1, tmp));
1241       break;
1242
1243     case TLS_MODEL_INITIAL_EXEC:
1244       addend_lo = ((addend & 0x3fff) ^ 0x2000) - 0x2000;
1245       addend_hi = addend - addend_lo;
1246
1247       op1 = plus_constant (Pmode, op1, addend_hi);
1248       addend = addend_lo;
1249
1250       tmp = gen_reg_rtx (Pmode);
1251       emit_insn (gen_load_tprel (tmp, op1));
1252
1253       if (!register_operand (op0, Pmode))
1254         op0 = gen_reg_rtx (Pmode);
1255       emit_insn (gen_adddi3 (op0, tmp, gen_thread_pointer ()));
1256       break;
1257
1258     case TLS_MODEL_LOCAL_EXEC:
1259       if (!register_operand (op0, Pmode))
1260         op0 = gen_reg_rtx (Pmode);
1261
1262       op1 = orig_op1;
1263       addend = 0;
1264       if (TARGET_TLS64)
1265         {
1266           emit_insn (gen_load_tprel (op0, op1));
1267           emit_insn (gen_adddi3 (op0, op0, gen_thread_pointer ()));
1268         }
1269       else
1270         emit_insn (gen_add_tprel (op0, op1, gen_thread_pointer ()));
1271       break;
1272
1273     default:
1274       gcc_unreachable ();
1275     }
1276
1277   if (addend)
1278     op0 = expand_simple_binop (Pmode, PLUS, op0, GEN_INT (addend),
1279                                orig_op0, 1, OPTAB_DIRECT);
1280   if (orig_op0 == op0)
1281     return NULL_RTX;
1282   if (GET_MODE (orig_op0) == Pmode)
1283     return op0;
1284   return gen_lowpart (GET_MODE (orig_op0), op0);
1285 }
1286
1287 rtx
1288 ia64_expand_move (rtx op0, rtx op1)
1289 {
1290   machine_mode mode = GET_MODE (op0);
1291
1292   if (!reload_in_progress && !reload_completed && !ia64_move_ok (op0, op1))
1293     op1 = force_reg (mode, op1);
1294
1295   if ((mode == Pmode || mode == ptr_mode) && symbolic_operand (op1, VOIDmode))
1296     {
1297       HOST_WIDE_INT addend = 0;
1298       enum tls_model tls_kind;
1299       rtx sym = op1;
1300
1301       if (GET_CODE (op1) == CONST
1302           && GET_CODE (XEXP (op1, 0)) == PLUS
1303           && GET_CODE (XEXP (XEXP (op1, 0), 1)) == CONST_INT)
1304         {
1305           addend = INTVAL (XEXP (XEXP (op1, 0), 1));
1306           sym = XEXP (XEXP (op1, 0), 0);
1307         }
1308
1309       tls_kind = tls_symbolic_operand_type (sym);
1310       if (tls_kind)
1311         return ia64_expand_tls_address (tls_kind, op0, sym, op1, addend);
1312
1313       if (any_offset_symbol_operand (sym, mode))
1314         addend = 0;
1315       else if (aligned_offset_symbol_operand (sym, mode))
1316         {
1317           HOST_WIDE_INT addend_lo, addend_hi;
1318
1319           addend_lo = ((addend & 0x3fff) ^ 0x2000) - 0x2000;
1320           addend_hi = addend - addend_lo;
1321
1322           if (addend_lo != 0)
1323             {
1324               op1 = plus_constant (mode, sym, addend_hi);
1325               addend = addend_lo;
1326             }
1327           else
1328             addend = 0;
1329         }
1330       else
1331         op1 = sym;
1332
1333       if (reload_completed)
1334         {
1335           /* We really should have taken care of this offset earlier.  */
1336           gcc_assert (addend == 0);
1337           if (ia64_expand_load_address (op0, op1))
1338             return NULL_RTX;
1339         }
1340
1341       if (addend)
1342         {
1343           rtx subtarget = !can_create_pseudo_p () ? op0 : gen_reg_rtx (mode);
1344
1345           emit_insn (gen_rtx_SET (VOIDmode, subtarget, op1));
1346
1347           op1 = expand_simple_binop (mode, PLUS, subtarget,
1348                                      GEN_INT (addend), op0, 1, OPTAB_DIRECT);
1349           if (op0 == op1)
1350             return NULL_RTX;
1351         }
1352     }
1353
1354   return op1;
1355 }
1356
1357 /* Split a move from OP1 to OP0 conditional on COND.  */
1358
1359 void
1360 ia64_emit_cond_move (rtx op0, rtx op1, rtx cond)
1361 {
1362   rtx_insn *insn, *first = get_last_insn ();
1363
1364   emit_move_insn (op0, op1);
1365
1366   for (insn = get_last_insn (); insn != first; insn = PREV_INSN (insn))
1367     if (INSN_P (insn))
1368       PATTERN (insn) = gen_rtx_COND_EXEC (VOIDmode, copy_rtx (cond),
1369                                           PATTERN (insn));
1370 }
1371
1372 /* Split a post-reload TImode or TFmode reference into two DImode
1373    components.  This is made extra difficult by the fact that we do
1374    not get any scratch registers to work with, because reload cannot
1375    be prevented from giving us a scratch that overlaps the register
1376    pair involved.  So instead, when addressing memory, we tweak the
1377    pointer register up and back down with POST_INCs.  Or up and not
1378    back down when we can get away with it.
1379
1380    REVERSED is true when the loads must be done in reversed order
1381    (high word first) for correctness.  DEAD is true when the pointer
1382    dies with the second insn we generate and therefore the second
1383    address must not carry a postmodify.
1384
1385    May return an insn which is to be emitted after the moves.  */
1386
1387 static rtx
1388 ia64_split_tmode (rtx out[2], rtx in, bool reversed, bool dead)
1389 {
1390   rtx fixup = 0;
1391
1392   switch (GET_CODE (in))
1393     {
1394     case REG:
1395       out[reversed] = gen_rtx_REG (DImode, REGNO (in));
1396       out[!reversed] = gen_rtx_REG (DImode, REGNO (in) + 1);
1397       break;
1398
1399     case CONST_INT:
1400     case CONST_DOUBLE:
1401       /* Cannot occur reversed.  */
1402       gcc_assert (!reversed);
1403
1404       if (GET_MODE (in) != TFmode)
1405         split_double (in, &out[0], &out[1]);
1406       else
1407         /* split_double does not understand how to split a TFmode
1408            quantity into a pair of DImode constants.  */
1409         {
1410           REAL_VALUE_TYPE r;
1411           unsigned HOST_WIDE_INT p[2];
1412           long l[4];  /* TFmode is 128 bits */
1413
1414           REAL_VALUE_FROM_CONST_DOUBLE (r, in);
1415           real_to_target (l, &r, TFmode);
1416
1417           if (FLOAT_WORDS_BIG_ENDIAN)
1418             {
1419               p[0] = (((unsigned HOST_WIDE_INT) l[0]) << 32) + l[1];
1420               p[1] = (((unsigned HOST_WIDE_INT) l[2]) << 32) + l[3];
1421             }
1422           else
1423             {
1424               p[0] = (((unsigned HOST_WIDE_INT) l[1]) << 32) + l[0];
1425               p[1] = (((unsigned HOST_WIDE_INT) l[3]) << 32) + l[2];
1426             }
1427           out[0] = GEN_INT (p[0]);
1428           out[1] = GEN_INT (p[1]);
1429         }
1430       break;
1431
1432     case MEM:
1433       {
1434         rtx base = XEXP (in, 0);
1435         rtx offset;
1436
1437         switch (GET_CODE (base))
1438           {
1439           case REG:
1440             if (!reversed)
1441               {
1442                 out[0] = adjust_automodify_address
1443                   (in, DImode, gen_rtx_POST_INC (Pmode, base), 0);
1444                 out[1] = adjust_automodify_address
1445                   (in, DImode, dead ? 0 : gen_rtx_POST_DEC (Pmode, base), 8);
1446               }
1447             else
1448               {
1449                 /* Reversal requires a pre-increment, which can only
1450                    be done as a separate insn.  */
1451                 emit_insn (gen_adddi3 (base, base, GEN_INT (8)));
1452                 out[0] = adjust_automodify_address
1453                   (in, DImode, gen_rtx_POST_DEC (Pmode, base), 8);
1454                 out[1] = adjust_address (in, DImode, 0);
1455               }
1456             break;
1457
1458           case POST_INC:
1459             gcc_assert (!reversed && !dead);
1460
1461             /* Just do the increment in two steps.  */
1462             out[0] = adjust_automodify_address (in, DImode, 0, 0);
1463             out[1] = adjust_automodify_address (in, DImode, 0, 8);
1464             break;
1465
1466           case POST_DEC:
1467             gcc_assert (!reversed && !dead);
1468
1469             /* Add 8, subtract 24.  */
1470             base = XEXP (base, 0);
1471             out[0] = adjust_automodify_address
1472               (in, DImode, gen_rtx_POST_INC (Pmode, base), 0);
1473             out[1] = adjust_automodify_address
1474               (in, DImode,
1475                gen_rtx_POST_MODIFY (Pmode, base,
1476                                     plus_constant (Pmode, base, -24)),
1477                8);
1478             break;
1479
1480           case POST_MODIFY:
1481             gcc_assert (!reversed && !dead);
1482
1483             /* Extract and adjust the modification.  This case is
1484                trickier than the others, because we might have an
1485                index register, or we might have a combined offset that
1486                doesn't fit a signed 9-bit displacement field.  We can
1487                assume the incoming expression is already legitimate.  */
1488             offset = XEXP (base, 1);
1489             base = XEXP (base, 0);
1490
1491             out[0] = adjust_automodify_address
1492               (in, DImode, gen_rtx_POST_INC (Pmode, base), 0);
1493
1494             if (GET_CODE (XEXP (offset, 1)) == REG)
1495               {
1496                 /* Can't adjust the postmodify to match.  Emit the
1497                    original, then a separate addition insn.  */
1498                 out[1] = adjust_automodify_address (in, DImode, 0, 8);
1499                 fixup = gen_adddi3 (base, base, GEN_INT (-8));
1500               }
1501             else
1502               {
1503                 gcc_assert (GET_CODE (XEXP (offset, 1)) == CONST_INT);
1504                 if (INTVAL (XEXP (offset, 1)) < -256 + 8)
1505                   {
1506                     /* Again the postmodify cannot be made to match,
1507                        but in this case it's more efficient to get rid
1508                        of the postmodify entirely and fix up with an
1509                        add insn.  */
1510                     out[1] = adjust_automodify_address (in, DImode, base, 8);
1511                     fixup = gen_adddi3
1512                       (base, base, GEN_INT (INTVAL (XEXP (offset, 1)) - 8));
1513                   }
1514                 else
1515                   {
1516                     /* Combined offset still fits in the displacement field.
1517                        (We cannot overflow it at the high end.)  */
1518                     out[1] = adjust_automodify_address
1519                       (in, DImode, gen_rtx_POST_MODIFY
1520                        (Pmode, base, gen_rtx_PLUS
1521                         (Pmode, base,
1522                          GEN_INT (INTVAL (XEXP (offset, 1)) - 8))),
1523                        8);
1524                   }
1525               }
1526             break;
1527
1528           default:
1529             gcc_unreachable ();
1530           }
1531         break;
1532       }
1533
1534     default:
1535       gcc_unreachable ();
1536     }
1537
1538   return fixup;
1539 }
1540
1541 /* Split a TImode or TFmode move instruction after reload.
1542    This is used by *movtf_internal and *movti_internal.  */
1543 void
1544 ia64_split_tmode_move (rtx operands[])
1545 {
1546   rtx in[2], out[2], insn;
1547   rtx fixup[2];
1548   bool dead = false;
1549   bool reversed = false;
1550
1551   /* It is possible for reload to decide to overwrite a pointer with
1552      the value it points to.  In that case we have to do the loads in
1553      the appropriate order so that the pointer is not destroyed too
1554      early.  Also we must not generate a postmodify for that second
1555      load, or rws_access_regno will die.  And we must not generate a
1556      postmodify for the second load if the destination register
1557      overlaps with the base register.  */
1558   if (GET_CODE (operands[1]) == MEM
1559       && reg_overlap_mentioned_p (operands[0], operands[1]))
1560     {
1561       rtx base = XEXP (operands[1], 0);
1562       while (GET_CODE (base) != REG)
1563         base = XEXP (base, 0);
1564
1565       if (REGNO (base) == REGNO (operands[0]))
1566         reversed = true;
1567
1568       if (refers_to_regno_p (REGNO (operands[0]),
1569                              REGNO (operands[0])+2,
1570                              base, 0))
1571         dead = true;
1572     }
1573   /* Another reason to do the moves in reversed order is if the first
1574      element of the target register pair is also the second element of
1575      the source register pair.  */
1576   if (GET_CODE (operands[0]) == REG && GET_CODE (operands[1]) == REG
1577       && REGNO (operands[0]) == REGNO (operands[1]) + 1)
1578     reversed = true;
1579
1580   fixup[0] = ia64_split_tmode (in, operands[1], reversed, dead);
1581   fixup[1] = ia64_split_tmode (out, operands[0], reversed, dead);
1582
1583 #define MAYBE_ADD_REG_INC_NOTE(INSN, EXP)                               \
1584   if (GET_CODE (EXP) == MEM                                             \
1585       && (GET_CODE (XEXP (EXP, 0)) == POST_MODIFY                       \
1586           || GET_CODE (XEXP (EXP, 0)) == POST_INC                       \
1587           || GET_CODE (XEXP (EXP, 0)) == POST_DEC))                     \
1588     add_reg_note (insn, REG_INC, XEXP (XEXP (EXP, 0), 0))
1589
1590   insn = emit_insn (gen_rtx_SET (VOIDmode, out[0], in[0]));
1591   MAYBE_ADD_REG_INC_NOTE (insn, in[0]);
1592   MAYBE_ADD_REG_INC_NOTE (insn, out[0]);
1593
1594   insn = emit_insn (gen_rtx_SET (VOIDmode, out[1], in[1]));
1595   MAYBE_ADD_REG_INC_NOTE (insn, in[1]);
1596   MAYBE_ADD_REG_INC_NOTE (insn, out[1]);
1597
1598   if (fixup[0])
1599     emit_insn (fixup[0]);
1600   if (fixup[1])
1601     emit_insn (fixup[1]);
1602
1603 #undef MAYBE_ADD_REG_INC_NOTE
1604 }
1605
1606 /* ??? Fixing GR->FR XFmode moves during reload is hard.  You need to go
1607    through memory plus an extra GR scratch register.  Except that you can
1608    either get the first from SECONDARY_MEMORY_NEEDED or the second from
1609    SECONDARY_RELOAD_CLASS, but not both.
1610
1611    We got into problems in the first place by allowing a construct like
1612    (subreg:XF (reg:TI)), which we got from a union containing a long double.
1613    This solution attempts to prevent this situation from occurring.  When
1614    we see something like the above, we spill the inner register to memory.  */
1615
1616 static rtx
1617 spill_xfmode_rfmode_operand (rtx in, int force, machine_mode mode)
1618 {
1619   if (GET_CODE (in) == SUBREG
1620       && GET_MODE (SUBREG_REG (in)) == TImode
1621       && GET_CODE (SUBREG_REG (in)) == REG)
1622     {
1623       rtx memt = assign_stack_temp (TImode, 16);
1624       emit_move_insn (memt, SUBREG_REG (in));
1625       return adjust_address (memt, mode, 0);
1626     }
1627   else if (force && GET_CODE (in) == REG)
1628     {
1629       rtx memx = assign_stack_temp (mode, 16);
1630       emit_move_insn (memx, in);
1631       return memx;
1632     }
1633   else
1634     return in;
1635 }
1636
1637 /* Expand the movxf or movrf pattern (MODE says which) with the given
1638    OPERANDS, returning true if the pattern should then invoke
1639    DONE.  */
1640
1641 bool
1642 ia64_expand_movxf_movrf (machine_mode mode, rtx operands[])
1643 {
1644   rtx op0 = operands[0];
1645
1646   if (GET_CODE (op0) == SUBREG)
1647     op0 = SUBREG_REG (op0);
1648
1649   /* We must support XFmode loads into general registers for stdarg/vararg,
1650      unprototyped calls, and a rare case where a long double is passed as
1651      an argument after a float HFA fills the FP registers.  We split them into
1652      DImode loads for convenience.  We also need to support XFmode stores
1653      for the last case.  This case does not happen for stdarg/vararg routines,
1654      because we do a block store to memory of unnamed arguments.  */
1655
1656   if (GET_CODE (op0) == REG && GR_REGNO_P (REGNO (op0)))
1657     {
1658       rtx out[2];
1659
1660       /* We're hoping to transform everything that deals with XFmode
1661          quantities and GR registers early in the compiler.  */
1662       gcc_assert (can_create_pseudo_p ());
1663
1664       /* Struct to register can just use TImode instead.  */
1665       if ((GET_CODE (operands[1]) == SUBREG
1666            && GET_MODE (SUBREG_REG (operands[1])) == TImode)
1667           || (GET_CODE (operands[1]) == REG
1668               && GR_REGNO_P (REGNO (operands[1]))))
1669         {
1670           rtx op1 = operands[1];
1671
1672           if (GET_CODE (op1) == SUBREG)
1673             op1 = SUBREG_REG (op1);
1674           else
1675             op1 = gen_rtx_REG (TImode, REGNO (op1));
1676
1677           emit_move_insn (gen_rtx_REG (TImode, REGNO (op0)), op1);
1678           return true;
1679         }
1680
1681       if (GET_CODE (operands[1]) == CONST_DOUBLE)
1682         {
1683           /* Don't word-swap when reading in the constant.  */
1684           emit_move_insn (gen_rtx_REG (DImode, REGNO (op0)),
1685                           operand_subword (operands[1], WORDS_BIG_ENDIAN,
1686                                            0, mode));
1687           emit_move_insn (gen_rtx_REG (DImode, REGNO (op0) + 1),
1688                           operand_subword (operands[1], !WORDS_BIG_ENDIAN,
1689                                            0, mode));
1690           return true;
1691         }
1692
1693       /* If the quantity is in a register not known to be GR, spill it.  */
1694       if (register_operand (operands[1], mode))
1695         operands[1] = spill_xfmode_rfmode_operand (operands[1], 1, mode);
1696
1697       gcc_assert (GET_CODE (operands[1]) == MEM);
1698
1699       /* Don't word-swap when reading in the value.  */
1700       out[0] = gen_rtx_REG (DImode, REGNO (op0));
1701       out[1] = gen_rtx_REG (DImode, REGNO (op0) + 1);
1702
1703       emit_move_insn (out[0], adjust_address (operands[1], DImode, 0));
1704       emit_move_insn (out[1], adjust_address (operands[1], DImode, 8));
1705       return true;
1706     }
1707
1708   if (GET_CODE (operands[1]) == REG && GR_REGNO_P (REGNO (operands[1])))
1709     {
1710       /* We're hoping to transform everything that deals with XFmode
1711          quantities and GR registers early in the compiler.  */
1712       gcc_assert (can_create_pseudo_p ());
1713
1714       /* Op0 can't be a GR_REG here, as that case is handled above.
1715          If op0 is a register, then we spill op1, so that we now have a
1716          MEM operand.  This requires creating an XFmode subreg of a TImode reg
1717          to force the spill.  */
1718       if (register_operand (operands[0], mode))
1719         {
1720           rtx op1 = gen_rtx_REG (TImode, REGNO (operands[1]));
1721           op1 = gen_rtx_SUBREG (mode, op1, 0);
1722           operands[1] = spill_xfmode_rfmode_operand (op1, 0, mode);
1723         }
1724
1725       else
1726         {
1727           rtx in[2];
1728
1729           gcc_assert (GET_CODE (operands[0]) == MEM);
1730
1731           /* Don't word-swap when writing out the value.  */
1732           in[0] = gen_rtx_REG (DImode, REGNO (operands[1]));
1733           in[1] = gen_rtx_REG (DImode, REGNO (operands[1]) + 1);
1734
1735           emit_move_insn (adjust_address (operands[0], DImode, 0), in[0]);
1736           emit_move_insn (adjust_address (operands[0], DImode, 8), in[1]);
1737           return true;
1738         }
1739     }
1740
1741   if (!reload_in_progress && !reload_completed)
1742     {
1743       operands[1] = spill_xfmode_rfmode_operand (operands[1], 0, mode);
1744
1745       if (GET_MODE (op0) == TImode && GET_CODE (op0) == REG)
1746         {
1747           rtx memt, memx, in = operands[1];
1748           if (CONSTANT_P (in))
1749             in = validize_mem (force_const_mem (mode, in));
1750           if (GET_CODE (in) == MEM)
1751             memt = adjust_address (in, TImode, 0);
1752           else
1753             {
1754               memt = assign_stack_temp (TImode, 16);
1755               memx = adjust_address (memt, mode, 0);
1756               emit_move_insn (memx, in);
1757             }
1758           emit_move_insn (op0, memt);
1759           return true;
1760         }
1761
1762       if (!ia64_move_ok (operands[0], operands[1]))
1763         operands[1] = force_reg (mode, operands[1]);
1764     }
1765
1766   return false;
1767 }
1768
1769 /* Emit comparison instruction if necessary, replacing *EXPR, *OP0, *OP1
1770    with the expression that holds the compare result (in VOIDmode).  */
1771
1772 static GTY(()) rtx cmptf_libfunc;
1773
1774 void
1775 ia64_expand_compare (rtx *expr, rtx *op0, rtx *op1)
1776 {
1777   enum rtx_code code = GET_CODE (*expr);
1778   rtx cmp;
1779
1780   /* If we have a BImode input, then we already have a compare result, and
1781      do not need to emit another comparison.  */
1782   if (GET_MODE (*op0) == BImode)
1783     {
1784       gcc_assert ((code == NE || code == EQ) && *op1 == const0_rtx);
1785       cmp = *op0;
1786     }
1787   /* HPUX TFmode compare requires a library call to _U_Qfcmp, which takes a
1788      magic number as its third argument, that indicates what to do.
1789      The return value is an integer to be compared against zero.  */
1790   else if (TARGET_HPUX && GET_MODE (*op0) == TFmode)
1791     {
1792       enum qfcmp_magic {
1793         QCMP_INV = 1,   /* Raise FP_INVALID on NaNs as a side effect.  */
1794         QCMP_UNORD = 2,
1795         QCMP_EQ = 4,
1796         QCMP_LT = 8,
1797         QCMP_GT = 16
1798       };
1799       int magic;
1800       enum rtx_code ncode;
1801       rtx ret, insns;
1802
1803       gcc_assert (cmptf_libfunc && GET_MODE (*op1) == TFmode);
1804       switch (code)
1805         {
1806           /* 1 = equal, 0 = not equal.  Equality operators do
1807              not raise FP_INVALID when given a NaN operand.  */
1808         case EQ:        magic = QCMP_EQ;                  ncode = NE; break;
1809         case NE:        magic = QCMP_EQ;                  ncode = EQ; break;
1810           /* isunordered() from C99.  */
1811         case UNORDERED: magic = QCMP_UNORD;               ncode = NE; break;
1812         case ORDERED:   magic = QCMP_UNORD;               ncode = EQ; break;
1813           /* Relational operators raise FP_INVALID when given
1814              a NaN operand.  */
1815         case LT:        magic = QCMP_LT        |QCMP_INV; ncode = NE; break;
1816         case LE:        magic = QCMP_LT|QCMP_EQ|QCMP_INV; ncode = NE; break;
1817         case GT:        magic = QCMP_GT        |QCMP_INV; ncode = NE; break;
1818         case GE:        magic = QCMP_GT|QCMP_EQ|QCMP_INV; ncode = NE; break;
1819           /* Unordered relational operators do not raise FP_INVALID
1820              when given a NaN operand.  */
1821         case UNLT:    magic = QCMP_LT        |QCMP_UNORD; ncode = NE; break;
1822         case UNLE:    magic = QCMP_LT|QCMP_EQ|QCMP_UNORD; ncode = NE; break;
1823         case UNGT:    magic = QCMP_GT        |QCMP_UNORD; ncode = NE; break;
1824         case UNGE:    magic = QCMP_GT|QCMP_EQ|QCMP_UNORD; ncode = NE; break;
1825           /* Not supported.  */
1826         case UNEQ:
1827         case LTGT:
1828         default: gcc_unreachable ();
1829         }
1830
1831       start_sequence ();
1832
1833       ret = emit_library_call_value (cmptf_libfunc, 0, LCT_CONST, DImode, 3,
1834                                      *op0, TFmode, *op1, TFmode,
1835                                      GEN_INT (magic), DImode);
1836       cmp = gen_reg_rtx (BImode);
1837       emit_insn (gen_rtx_SET (VOIDmode, cmp,
1838                               gen_rtx_fmt_ee (ncode, BImode,
1839                                               ret, const0_rtx)));
1840
1841       insns = get_insns ();
1842       end_sequence ();
1843
1844       emit_libcall_block (insns, cmp, cmp,
1845                           gen_rtx_fmt_ee (code, BImode, *op0, *op1));
1846       code = NE;
1847     }
1848   else
1849     {
1850       cmp = gen_reg_rtx (BImode);
1851       emit_insn (gen_rtx_SET (VOIDmode, cmp,
1852                               gen_rtx_fmt_ee (code, BImode, *op0, *op1)));
1853       code = NE;
1854     }
1855
1856   *expr = gen_rtx_fmt_ee (code, VOIDmode, cmp, const0_rtx);
1857   *op0 = cmp;
1858   *op1 = const0_rtx;
1859 }
1860
1861 /* Generate an integral vector comparison.  Return true if the condition has
1862    been reversed, and so the sense of the comparison should be inverted.  */
1863
1864 static bool
1865 ia64_expand_vecint_compare (enum rtx_code code, machine_mode mode,
1866                             rtx dest, rtx op0, rtx op1)
1867 {
1868   bool negate = false;
1869   rtx x;
1870
1871   /* Canonicalize the comparison to EQ, GT, GTU.  */
1872   switch (code)
1873     {
1874     case EQ:
1875     case GT:
1876     case GTU:
1877       break;
1878
1879     case NE:
1880     case LE:
1881     case LEU:
1882       code = reverse_condition (code);
1883       negate = true;
1884       break;
1885
1886     case GE:
1887     case GEU:
1888       code = reverse_condition (code);
1889       negate = true;
1890       /* FALLTHRU */
1891
1892     case LT:
1893     case LTU:
1894       code = swap_condition (code);
1895       x = op0, op0 = op1, op1 = x;
1896       break;
1897
1898     default:
1899       gcc_unreachable ();
1900     }
1901
1902   /* Unsigned parallel compare is not supported by the hardware.  Play some
1903      tricks to turn this into a signed comparison against 0.  */
1904   if (code == GTU)
1905     {
1906       switch (mode)
1907         {
1908         case V2SImode:
1909           {
1910             rtx t1, t2, mask;
1911
1912             /* Subtract (-(INT MAX) - 1) from both operands to make
1913                them signed.  */
1914             mask = GEN_INT (0x80000000);
1915             mask = gen_rtx_CONST_VECTOR (V2SImode, gen_rtvec (2, mask, mask));
1916             mask = force_reg (mode, mask);
1917             t1 = gen_reg_rtx (mode);
1918             emit_insn (gen_subv2si3 (t1, op0, mask));
1919             t2 = gen_reg_rtx (mode);
1920             emit_insn (gen_subv2si3 (t2, op1, mask));
1921             op0 = t1;
1922             op1 = t2;
1923             code = GT;
1924           }
1925           break;
1926
1927         case V8QImode:
1928         case V4HImode:
1929           /* Perform a parallel unsigned saturating subtraction.  */
1930           x = gen_reg_rtx (mode);
1931           emit_insn (gen_rtx_SET (VOIDmode, x,
1932                                   gen_rtx_US_MINUS (mode, op0, op1)));
1933
1934           code = EQ;
1935           op0 = x;
1936           op1 = CONST0_RTX (mode);
1937           negate = !negate;
1938           break;
1939
1940         default:
1941           gcc_unreachable ();
1942         }
1943     }
1944
1945   x = gen_rtx_fmt_ee (code, mode, op0, op1);
1946   emit_insn (gen_rtx_SET (VOIDmode, dest, x));
1947
1948   return negate;
1949 }
1950
1951 /* Emit an integral vector conditional move.  */
1952
1953 void
1954 ia64_expand_vecint_cmov (rtx operands[])
1955 {
1956   machine_mode mode = GET_MODE (operands[0]);
1957   enum rtx_code code = GET_CODE (operands[3]);
1958   bool negate;
1959   rtx cmp, x, ot, of;
1960
1961   cmp = gen_reg_rtx (mode);
1962   negate = ia64_expand_vecint_compare (code, mode, cmp,
1963                                        operands[4], operands[5]);
1964
1965   ot = operands[1+negate];
1966   of = operands[2-negate];
1967
1968   if (ot == CONST0_RTX (mode))
1969     {
1970       if (of == CONST0_RTX (mode))
1971         {
1972           emit_move_insn (operands[0], ot);
1973           return;
1974         }
1975
1976       x = gen_rtx_NOT (mode, cmp);
1977       x = gen_rtx_AND (mode, x, of);
1978       emit_insn (gen_rtx_SET (VOIDmode, operands[0], x));
1979     }
1980   else if (of == CONST0_RTX (mode))
1981     {
1982       x = gen_rtx_AND (mode, cmp, ot);
1983       emit_insn (gen_rtx_SET (VOIDmode, operands[0], x));
1984     }
1985   else
1986     {
1987       rtx t, f;
1988
1989       t = gen_reg_rtx (mode);
1990       x = gen_rtx_AND (mode, cmp, operands[1+negate]);
1991       emit_insn (gen_rtx_SET (VOIDmode, t, x));
1992
1993       f = gen_reg_rtx (mode);
1994       x = gen_rtx_NOT (mode, cmp);
1995       x = gen_rtx_AND (mode, x, operands[2-negate]);
1996       emit_insn (gen_rtx_SET (VOIDmode, f, x));
1997
1998       x = gen_rtx_IOR (mode, t, f);
1999       emit_insn (gen_rtx_SET (VOIDmode, operands[0], x));
2000     }
2001 }
2002
2003 /* Emit an integral vector min or max operation.  Return true if all done.  */
2004
2005 bool
2006 ia64_expand_vecint_minmax (enum rtx_code code, machine_mode mode,
2007                            rtx operands[])
2008 {
2009   rtx xops[6];
2010
2011   /* These four combinations are supported directly.  */
2012   if (mode == V8QImode && (code == UMIN || code == UMAX))
2013     return false;
2014   if (mode == V4HImode && (code == SMIN || code == SMAX))
2015     return false;
2016
2017   /* This combination can be implemented with only saturating subtraction.  */
2018   if (mode == V4HImode && code == UMAX)
2019     {
2020       rtx x, tmp = gen_reg_rtx (mode);
2021
2022       x = gen_rtx_US_MINUS (mode, operands[1], operands[2]);
2023       emit_insn (gen_rtx_SET (VOIDmode, tmp, x));
2024
2025       emit_insn (gen_addv4hi3 (operands[0], tmp, operands[2]));
2026       return true;
2027     }
2028
2029   /* Everything else implemented via vector comparisons.  */
2030   xops[0] = operands[0];
2031   xops[4] = xops[1] = operands[1];
2032   xops[5] = xops[2] = operands[2];
2033
2034   switch (code)
2035     {
2036     case UMIN:
2037       code = LTU;
2038       break;
2039     case UMAX:
2040       code = GTU;
2041       break;
2042     case SMIN:
2043       code = LT;
2044       break;
2045     case SMAX:
2046       code = GT;
2047       break;
2048     default:
2049       gcc_unreachable ();
2050     }
2051   xops[3] = gen_rtx_fmt_ee (code, VOIDmode, operands[1], operands[2]);
2052
2053   ia64_expand_vecint_cmov (xops);
2054   return true;
2055 }
2056
2057 /* The vectors LO and HI each contain N halves of a double-wide vector.
2058    Reassemble either the first N/2 or the second N/2 elements.  */
2059
2060 void
2061 ia64_unpack_assemble (rtx out, rtx lo, rtx hi, bool highp)
2062 {
2063   machine_mode vmode = GET_MODE (lo);
2064   unsigned int i, high, nelt = GET_MODE_NUNITS (vmode);
2065   struct expand_vec_perm_d d;
2066   bool ok;
2067
2068   d.target = gen_lowpart (vmode, out);
2069   d.op0 = (TARGET_BIG_ENDIAN ? hi : lo);
2070   d.op1 = (TARGET_BIG_ENDIAN ? lo : hi);
2071   d.vmode = vmode;
2072   d.nelt = nelt;
2073   d.one_operand_p = false;
2074   d.testing_p = false;
2075
2076   high = (highp ? nelt / 2 : 0);
2077   for (i = 0; i < nelt / 2; ++i)
2078     {
2079       d.perm[i * 2] = i + high;
2080       d.perm[i * 2 + 1] = i + high + nelt;
2081     }
2082
2083   ok = ia64_expand_vec_perm_const_1 (&d);
2084   gcc_assert (ok);
2085 }
2086
2087 /* Return a vector of the sign-extension of VEC.  */
2088
2089 static rtx
2090 ia64_unpack_sign (rtx vec, bool unsignedp)
2091 {
2092   machine_mode mode = GET_MODE (vec);
2093   rtx zero = CONST0_RTX (mode);
2094
2095   if (unsignedp)
2096     return zero;
2097   else
2098     {
2099       rtx sign = gen_reg_rtx (mode);
2100       bool neg;
2101
2102       neg = ia64_expand_vecint_compare (LT, mode, sign, vec, zero);
2103       gcc_assert (!neg);
2104
2105       return sign;
2106     }
2107 }
2108
2109 /* Emit an integral vector unpack operation.  */
2110
2111 void
2112 ia64_expand_unpack (rtx operands[3], bool unsignedp, bool highp)
2113 {
2114   rtx sign = ia64_unpack_sign (operands[1], unsignedp);
2115   ia64_unpack_assemble (operands[0], operands[1], sign, highp);
2116 }
2117
2118 /* Emit an integral vector widening sum operations.  */
2119
2120 void
2121 ia64_expand_widen_sum (rtx operands[3], bool unsignedp)
2122 {
2123   machine_mode wmode;
2124   rtx l, h, t, sign;
2125
2126   sign = ia64_unpack_sign (operands[1], unsignedp);
2127
2128   wmode = GET_MODE (operands[0]);
2129   l = gen_reg_rtx (wmode);
2130   h = gen_reg_rtx (wmode);
2131
2132   ia64_unpack_assemble (l, operands[1], sign, false);
2133   ia64_unpack_assemble (h, operands[1], sign, true);
2134
2135   t = expand_binop (wmode, add_optab, l, operands[2], NULL, 0, OPTAB_DIRECT);
2136   t = expand_binop (wmode, add_optab, h, t, operands[0], 0, OPTAB_DIRECT);
2137   if (t != operands[0])
2138     emit_move_insn (operands[0], t);
2139 }
2140
2141 /* Emit the appropriate sequence for a call.  */
2142
2143 void
2144 ia64_expand_call (rtx retval, rtx addr, rtx nextarg ATTRIBUTE_UNUSED,
2145                   int sibcall_p)
2146 {
2147   rtx insn, b0;
2148
2149   addr = XEXP (addr, 0);
2150   addr = convert_memory_address (DImode, addr);
2151   b0 = gen_rtx_REG (DImode, R_BR (0));
2152
2153   /* ??? Should do this for functions known to bind local too.  */
2154   if (TARGET_NO_PIC || TARGET_AUTO_PIC)
2155     {
2156       if (sibcall_p)
2157         insn = gen_sibcall_nogp (addr);
2158       else if (! retval)
2159         insn = gen_call_nogp (addr, b0);
2160       else
2161         insn = gen_call_value_nogp (retval, addr, b0);
2162       insn = emit_call_insn (insn);
2163     }
2164   else
2165     {
2166       if (sibcall_p)
2167         insn = gen_sibcall_gp (addr);
2168       else if (! retval)
2169         insn = gen_call_gp (addr, b0);
2170       else
2171         insn = gen_call_value_gp (retval, addr, b0);
2172       insn = emit_call_insn (insn);
2173
2174       use_reg (&CALL_INSN_FUNCTION_USAGE (insn), pic_offset_table_rtx);
2175     }
2176
2177   if (sibcall_p)
2178     use_reg (&CALL_INSN_FUNCTION_USAGE (insn), b0);
2179
2180   if (TARGET_ABI_OPEN_VMS)
2181     use_reg (&CALL_INSN_FUNCTION_USAGE (insn),
2182              gen_rtx_REG (DImode, GR_REG (25)));
2183 }
2184
2185 static void
2186 reg_emitted (enum ia64_frame_regs r)
2187 {
2188   if (emitted_frame_related_regs[r] == 0)
2189     emitted_frame_related_regs[r] = current_frame_info.r[r];
2190   else
2191     gcc_assert (emitted_frame_related_regs[r] == current_frame_info.r[r]);
2192 }
2193
2194 static int
2195 get_reg (enum ia64_frame_regs r)
2196 {
2197   reg_emitted (r);
2198   return current_frame_info.r[r];
2199 }
2200
2201 static bool
2202 is_emitted (int regno)
2203 {
2204   unsigned int r;
2205
2206   for (r = reg_fp; r < number_of_ia64_frame_regs; r++)
2207     if (emitted_frame_related_regs[r] == regno)
2208       return true;
2209   return false;
2210 }
2211
2212 void
2213 ia64_reload_gp (void)
2214 {
2215   rtx tmp;
2216
2217   if (current_frame_info.r[reg_save_gp])
2218     {
2219       tmp = gen_rtx_REG (DImode, get_reg (reg_save_gp));
2220     }
2221   else
2222     {
2223       HOST_WIDE_INT offset;
2224       rtx offset_r;
2225
2226       offset = (current_frame_info.spill_cfa_off
2227                 + current_frame_info.spill_size);
2228       if (frame_pointer_needed)
2229         {
2230           tmp = hard_frame_pointer_rtx;
2231           offset = -offset;
2232         }
2233       else
2234         {
2235           tmp = stack_pointer_rtx;
2236           offset = current_frame_info.total_size - offset;
2237         }
2238
2239       offset_r = GEN_INT (offset);
2240       if (satisfies_constraint_I (offset_r))
2241         emit_insn (gen_adddi3 (pic_offset_table_rtx, tmp, offset_r));
2242       else
2243         {
2244           emit_move_insn (pic_offset_table_rtx, offset_r);
2245           emit_insn (gen_adddi3 (pic_offset_table_rtx,
2246                                  pic_offset_table_rtx, tmp));
2247         }
2248
2249       tmp = gen_rtx_MEM (DImode, pic_offset_table_rtx);
2250     }
2251
2252   emit_move_insn (pic_offset_table_rtx, tmp);
2253 }
2254
2255 void
2256 ia64_split_call (rtx retval, rtx addr, rtx retaddr, rtx scratch_r,
2257                  rtx scratch_b, int noreturn_p, int sibcall_p)
2258 {
2259   rtx insn;
2260   bool is_desc = false;
2261
2262   /* If we find we're calling through a register, then we're actually
2263      calling through a descriptor, so load up the values.  */
2264   if (REG_P (addr) && GR_REGNO_P (REGNO (addr)))
2265     {
2266       rtx tmp;
2267       bool addr_dead_p;
2268
2269       /* ??? We are currently constrained to *not* use peep2, because
2270          we can legitimately change the global lifetime of the GP
2271          (in the form of killing where previously live).  This is
2272          because a call through a descriptor doesn't use the previous
2273          value of the GP, while a direct call does, and we do not
2274          commit to either form until the split here.
2275
2276          That said, this means that we lack precise life info for
2277          whether ADDR is dead after this call.  This is not terribly
2278          important, since we can fix things up essentially for free
2279          with the POST_DEC below, but it's nice to not use it when we
2280          can immediately tell it's not necessary.  */
2281       addr_dead_p = ((noreturn_p || sibcall_p
2282                       || TEST_HARD_REG_BIT (regs_invalidated_by_call,
2283                                             REGNO (addr)))
2284                      && !FUNCTION_ARG_REGNO_P (REGNO (addr)));
2285
2286       /* Load the code address into scratch_b.  */
2287       tmp = gen_rtx_POST_INC (Pmode, addr);
2288       tmp = gen_rtx_MEM (Pmode, tmp);
2289       emit_move_insn (scratch_r, tmp);
2290       emit_move_insn (scratch_b, scratch_r);
2291
2292       /* Load the GP address.  If ADDR is not dead here, then we must
2293          revert the change made above via the POST_INCREMENT.  */
2294       if (!addr_dead_p)
2295         tmp = gen_rtx_POST_DEC (Pmode, addr);
2296       else
2297         tmp = addr;
2298       tmp = gen_rtx_MEM (Pmode, tmp);
2299       emit_move_insn (pic_offset_table_rtx, tmp);
2300
2301       is_desc = true;
2302       addr = scratch_b;
2303     }
2304
2305   if (sibcall_p)
2306     insn = gen_sibcall_nogp (addr);
2307   else if (retval)
2308     insn = gen_call_value_nogp (retval, addr, retaddr);
2309   else
2310     insn = gen_call_nogp (addr, retaddr);
2311   emit_call_insn (insn);
2312
2313   if ((!TARGET_CONST_GP || is_desc) && !noreturn_p && !sibcall_p)
2314     ia64_reload_gp ();
2315 }
2316
2317 /* Expand an atomic operation.  We want to perform MEM <CODE>= VAL atomically.
2318
2319    This differs from the generic code in that we know about the zero-extending
2320    properties of cmpxchg, and the zero-extending requirements of ar.ccv.  We
2321    also know that ld.acq+cmpxchg.rel equals a full barrier.
2322
2323    The loop we want to generate looks like
2324
2325         cmp_reg = mem;
2326       label:
2327         old_reg = cmp_reg;
2328         new_reg = cmp_reg op val;
2329         cmp_reg = compare-and-swap(mem, old_reg, new_reg)
2330         if (cmp_reg != old_reg)
2331           goto label;
2332
2333    Note that we only do the plain load from memory once.  Subsequent
2334    iterations use the value loaded by the compare-and-swap pattern.  */
2335
2336 void
2337 ia64_expand_atomic_op (enum rtx_code code, rtx mem, rtx val,
2338                        rtx old_dst, rtx new_dst, enum memmodel model)
2339 {
2340   machine_mode mode = GET_MODE (mem);
2341   rtx old_reg, new_reg, cmp_reg, ar_ccv, label;
2342   enum insn_code icode;
2343
2344   /* Special case for using fetchadd.  */
2345   if ((mode == SImode || mode == DImode)
2346       && (code == PLUS || code == MINUS)
2347       && fetchadd_operand (val, mode))
2348     {
2349       if (code == MINUS)
2350         val = GEN_INT (-INTVAL (val));
2351
2352       if (!old_dst)
2353         old_dst = gen_reg_rtx (mode);
2354
2355       switch (model)
2356         {
2357         case MEMMODEL_ACQ_REL:
2358         case MEMMODEL_SEQ_CST:
2359           emit_insn (gen_memory_barrier ());
2360           /* FALLTHRU */
2361         case MEMMODEL_RELAXED:
2362         case MEMMODEL_ACQUIRE:
2363         case MEMMODEL_CONSUME:
2364           if (mode == SImode)
2365             icode = CODE_FOR_fetchadd_acq_si;
2366           else
2367             icode = CODE_FOR_fetchadd_acq_di;
2368           break;
2369         case MEMMODEL_RELEASE:
2370           if (mode == SImode)
2371             icode = CODE_FOR_fetchadd_rel_si;
2372           else
2373             icode = CODE_FOR_fetchadd_rel_di;
2374           break;
2375
2376         default:
2377           gcc_unreachable ();
2378         }
2379
2380       emit_insn (GEN_FCN (icode) (old_dst, mem, val));
2381
2382       if (new_dst)
2383         {
2384           new_reg = expand_simple_binop (mode, PLUS, old_dst, val, new_dst,
2385                                          true, OPTAB_WIDEN);
2386           if (new_reg != new_dst)
2387             emit_move_insn (new_dst, new_reg);
2388         }
2389       return;
2390     }
2391
2392   /* Because of the volatile mem read, we get an ld.acq, which is the
2393      front half of the full barrier.  The end half is the cmpxchg.rel.
2394      For relaxed and release memory models, we don't need this.  But we
2395      also don't bother trying to prevent it either.  */
2396   gcc_assert (model == MEMMODEL_RELAXED
2397               || model == MEMMODEL_RELEASE
2398               || MEM_VOLATILE_P (mem));
2399
2400   old_reg = gen_reg_rtx (DImode);
2401   cmp_reg = gen_reg_rtx (DImode);
2402   label = gen_label_rtx ();
2403
2404   if (mode != DImode)
2405     {
2406       val = simplify_gen_subreg (DImode, val, mode, 0);
2407       emit_insn (gen_extend_insn (cmp_reg, mem, DImode, mode, 1));
2408     }
2409   else
2410     emit_move_insn (cmp_reg, mem);
2411
2412   emit_label (label);
2413
2414   ar_ccv = gen_rtx_REG (DImode, AR_CCV_REGNUM);
2415   emit_move_insn (old_reg, cmp_reg);
2416   emit_move_insn (ar_ccv, cmp_reg);
2417
2418   if (old_dst)
2419     emit_move_insn (old_dst, gen_lowpart (mode, cmp_reg));
2420
2421   new_reg = cmp_reg;
2422   if (code == NOT)
2423     {
2424       new_reg = expand_simple_binop (DImode, AND, new_reg, val, NULL_RTX,
2425                                      true, OPTAB_DIRECT);
2426       new_reg = expand_simple_unop (DImode, code, new_reg, NULL_RTX, true);
2427     }
2428   else
2429     new_reg = expand_simple_binop (DImode, code, new_reg, val, NULL_RTX,
2430                                    true, OPTAB_DIRECT);
2431
2432   if (mode != DImode)
2433     new_reg = gen_lowpart (mode, new_reg);
2434   if (new_dst)
2435     emit_move_insn (new_dst, new_reg);
2436
2437   switch (model)
2438     {
2439     case MEMMODEL_RELAXED:
2440     case MEMMODEL_ACQUIRE:
2441     case MEMMODEL_CONSUME:
2442       switch (mode)
2443         {
2444         case QImode: icode = CODE_FOR_cmpxchg_acq_qi;  break;
2445         case HImode: icode = CODE_FOR_cmpxchg_acq_hi;  break;
2446         case SImode: icode = CODE_FOR_cmpxchg_acq_si;  break;
2447         case DImode: icode = CODE_FOR_cmpxchg_acq_di;  break;
2448         default:
2449           gcc_unreachable ();
2450         }
2451       break;
2452
2453     case MEMMODEL_RELEASE:
2454     case MEMMODEL_ACQ_REL:
2455     case MEMMODEL_SEQ_CST:
2456       switch (mode)
2457         {
2458         case QImode: icode = CODE_FOR_cmpxchg_rel_qi;  break;
2459         case HImode: icode = CODE_FOR_cmpxchg_rel_hi;  break;
2460         case SImode: icode = CODE_FOR_cmpxchg_rel_si;  break;
2461         case DImode: icode = CODE_FOR_cmpxchg_rel_di;  break;
2462         default:
2463           gcc_unreachable ();
2464         }
2465       break;
2466
2467     default:
2468       gcc_unreachable ();
2469     }
2470
2471   emit_insn (GEN_FCN (icode) (cmp_reg, mem, ar_ccv, new_reg));
2472
2473   emit_cmp_and_jump_insns (cmp_reg, old_reg, NE, NULL, DImode, true, label);
2474 }
2475 \f
2476 /* Begin the assembly file.  */
2477
2478 static void
2479 ia64_file_start (void)
2480 {
2481   default_file_start ();
2482   emit_safe_across_calls ();
2483 }
2484
2485 void
2486 emit_safe_across_calls (void)
2487 {
2488   unsigned int rs, re;
2489   int out_state;
2490
2491   rs = 1;
2492   out_state = 0;
2493   while (1)
2494     {
2495       while (rs < 64 && call_used_regs[PR_REG (rs)])
2496         rs++;
2497       if (rs >= 64)
2498         break;
2499       for (re = rs + 1; re < 64 && ! call_used_regs[PR_REG (re)]; re++)
2500         continue;
2501       if (out_state == 0)
2502         {
2503           fputs ("\t.pred.safe_across_calls ", asm_out_file);
2504           out_state = 1;
2505         }
2506       else
2507         fputc (',', asm_out_file);
2508       if (re == rs + 1)
2509         fprintf (asm_out_file, "p%u", rs);
2510       else
2511         fprintf (asm_out_file, "p%u-p%u", rs, re - 1);
2512       rs = re + 1;
2513     }
2514   if (out_state)
2515     fputc ('\n', asm_out_file);
2516 }
2517
2518 /* Globalize a declaration.  */
2519
2520 static void
2521 ia64_globalize_decl_name (FILE * stream, tree decl)
2522 {
2523   const char *name = XSTR (XEXP (DECL_RTL (decl), 0), 0);
2524   tree version_attr = lookup_attribute ("version_id", DECL_ATTRIBUTES (decl));
2525   if (version_attr)
2526     {
2527       tree v = TREE_VALUE (TREE_VALUE (version_attr));
2528       const char *p = TREE_STRING_POINTER (v);
2529       fprintf (stream, "\t.alias %s#, \"%s{%s}\"\n", name, name, p);
2530     }
2531   targetm.asm_out.globalize_label (stream, name);
2532   if (TREE_CODE (decl) == FUNCTION_DECL)
2533     ASM_OUTPUT_TYPE_DIRECTIVE (stream, name, "function");
2534 }
2535
2536 /* Helper function for ia64_compute_frame_size: find an appropriate general
2537    register to spill some special register to.  SPECIAL_SPILL_MASK contains
2538    bits in GR0 to GR31 that have already been allocated by this routine.
2539    TRY_LOCALS is true if we should attempt to locate a local regnum.  */
2540
2541 static int
2542 find_gr_spill (enum ia64_frame_regs r, int try_locals)
2543 {
2544   int regno;
2545
2546   if (emitted_frame_related_regs[r] != 0)
2547     {
2548       regno = emitted_frame_related_regs[r];
2549       if (regno >= LOC_REG (0) && regno < LOC_REG (80 - frame_pointer_needed)
2550           && current_frame_info.n_local_regs < regno - LOC_REG (0) + 1)
2551         current_frame_info.n_local_regs = regno - LOC_REG (0) + 1;
2552       else if (crtl->is_leaf
2553                && regno >= GR_REG (1) && regno <= GR_REG (31))
2554         current_frame_info.gr_used_mask |= 1 << regno;
2555
2556       return regno;
2557     }
2558
2559   /* If this is a leaf function, first try an otherwise unused
2560      call-clobbered register.  */
2561   if (crtl->is_leaf)
2562     {
2563       for (regno = GR_REG (1); regno <= GR_REG (31); regno++)
2564         if (! df_regs_ever_live_p (regno)
2565             && call_used_regs[regno]
2566             && ! fixed_regs[regno]
2567             && ! global_regs[regno]
2568             && ((current_frame_info.gr_used_mask >> regno) & 1) == 0
2569             && ! is_emitted (regno))
2570           {
2571             current_frame_info.gr_used_mask |= 1 << regno;
2572             return regno;
2573           }
2574     }
2575
2576   if (try_locals)
2577     {
2578       regno = current_frame_info.n_local_regs;
2579       /* If there is a frame pointer, then we can't use loc79, because
2580          that is HARD_FRAME_POINTER_REGNUM.  In particular, see the
2581          reg_name switching code in ia64_expand_prologue.  */
2582       while (regno < (80 - frame_pointer_needed))
2583         if (! is_emitted (LOC_REG (regno++)))
2584           {
2585             current_frame_info.n_local_regs = regno;
2586             return LOC_REG (regno - 1);
2587           }
2588     }
2589
2590   /* Failed to find a general register to spill to.  Must use stack.  */
2591   return 0;
2592 }
2593
2594 /* In order to make for nice schedules, we try to allocate every temporary
2595    to a different register.  We must of course stay away from call-saved,
2596    fixed, and global registers.  We must also stay away from registers
2597    allocated in current_frame_info.gr_used_mask, since those include regs
2598    used all through the prologue.
2599
2600    Any register allocated here must be used immediately.  The idea is to
2601    aid scheduling, not to solve data flow problems.  */
2602
2603 static int last_scratch_gr_reg;
2604
2605 static int
2606 next_scratch_gr_reg (void)
2607 {
2608   int i, regno;
2609
2610   for (i = 0; i < 32; ++i)
2611     {
2612       regno = (last_scratch_gr_reg + i + 1) & 31;
2613       if (call_used_regs[regno]
2614           && ! fixed_regs[regno]
2615           && ! global_regs[regno]
2616           && ((current_frame_info.gr_used_mask >> regno) & 1) == 0)
2617         {
2618           last_scratch_gr_reg = regno;
2619           return regno;
2620         }
2621     }
2622
2623   /* There must be _something_ available.  */
2624   gcc_unreachable ();
2625 }
2626
2627 /* Helper function for ia64_compute_frame_size, called through
2628    diddle_return_value.  Mark REG in current_frame_info.gr_used_mask.  */
2629
2630 static void
2631 mark_reg_gr_used_mask (rtx reg, void *data ATTRIBUTE_UNUSED)
2632 {
2633   unsigned int regno = REGNO (reg);
2634   if (regno < 32)
2635     {
2636       unsigned int i, n = hard_regno_nregs[regno][GET_MODE (reg)];
2637       for (i = 0; i < n; ++i)
2638         current_frame_info.gr_used_mask |= 1 << (regno + i);
2639     }
2640 }
2641
2642
2643 /* Returns the number of bytes offset between the frame pointer and the stack
2644    pointer for the current function.  SIZE is the number of bytes of space
2645    needed for local variables.  */
2646
2647 static void
2648 ia64_compute_frame_size (HOST_WIDE_INT size)
2649 {
2650   HOST_WIDE_INT total_size;
2651   HOST_WIDE_INT spill_size = 0;
2652   HOST_WIDE_INT extra_spill_size = 0;
2653   HOST_WIDE_INT pretend_args_size;
2654   HARD_REG_SET mask;
2655   int n_spilled = 0;
2656   int spilled_gr_p = 0;
2657   int spilled_fr_p = 0;
2658   unsigned int regno;
2659   int min_regno;
2660   int max_regno;
2661   int i;
2662
2663   if (current_frame_info.initialized)
2664     return;
2665
2666   memset (&current_frame_info, 0, sizeof current_frame_info);
2667   CLEAR_HARD_REG_SET (mask);
2668
2669   /* Don't allocate scratches to the return register.  */
2670   diddle_return_value (mark_reg_gr_used_mask, NULL);
2671
2672   /* Don't allocate scratches to the EH scratch registers.  */
2673   if (cfun->machine->ia64_eh_epilogue_sp)
2674     mark_reg_gr_used_mask (cfun->machine->ia64_eh_epilogue_sp, NULL);
2675   if (cfun->machine->ia64_eh_epilogue_bsp)
2676     mark_reg_gr_used_mask (cfun->machine->ia64_eh_epilogue_bsp, NULL);
2677
2678   /* Static stack checking uses r2 and r3.  */
2679   if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
2680     current_frame_info.gr_used_mask |= 0xc;
2681
2682   /* Find the size of the register stack frame.  We have only 80 local
2683      registers, because we reserve 8 for the inputs and 8 for the
2684      outputs.  */
2685
2686   /* Skip HARD_FRAME_POINTER_REGNUM (loc79) when frame_pointer_needed,
2687      since we'll be adjusting that down later.  */
2688   regno = LOC_REG (78) + ! frame_pointer_needed;
2689   for (; regno >= LOC_REG (0); regno--)
2690     if (df_regs_ever_live_p (regno) && !is_emitted (regno))
2691       break;
2692   current_frame_info.n_local_regs = regno - LOC_REG (0) + 1;
2693
2694   /* For functions marked with the syscall_linkage attribute, we must mark
2695      all eight input registers as in use, so that locals aren't visible to
2696      the caller.  */
2697
2698   if (cfun->machine->n_varargs > 0
2699       || lookup_attribute ("syscall_linkage",
2700                            TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
2701     current_frame_info.n_input_regs = 8;
2702   else
2703     {
2704       for (regno = IN_REG (7); regno >= IN_REG (0); regno--)
2705         if (df_regs_ever_live_p (regno))
2706           break;
2707       current_frame_info.n_input_regs = regno - IN_REG (0) + 1;
2708     }
2709
2710   for (regno = OUT_REG (7); regno >= OUT_REG (0); regno--)
2711     if (df_regs_ever_live_p (regno))
2712       break;
2713   i = regno - OUT_REG (0) + 1;
2714
2715 #ifndef PROFILE_HOOK
2716   /* When -p profiling, we need one output register for the mcount argument.
2717      Likewise for -a profiling for the bb_init_func argument.  For -ax
2718      profiling, we need two output registers for the two bb_init_trace_func
2719      arguments.  */
2720   if (crtl->profile)
2721     i = MAX (i, 1);
2722 #endif
2723   current_frame_info.n_output_regs = i;
2724
2725   /* ??? No rotating register support yet.  */
2726   current_frame_info.n_rotate_regs = 0;
2727
2728   /* Discover which registers need spilling, and how much room that
2729      will take.  Begin with floating point and general registers,
2730      which will always wind up on the stack.  */
2731
2732   for (regno = FR_REG (2); regno <= FR_REG (127); regno++)
2733     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2734       {
2735         SET_HARD_REG_BIT (mask, regno);
2736         spill_size += 16;
2737         n_spilled += 1;
2738         spilled_fr_p = 1;
2739       }
2740
2741   for (regno = GR_REG (1); regno <= GR_REG (31); regno++)
2742     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2743       {
2744         SET_HARD_REG_BIT (mask, regno);
2745         spill_size += 8;
2746         n_spilled += 1;
2747         spilled_gr_p = 1;
2748       }
2749
2750   for (regno = BR_REG (1); regno <= BR_REG (7); regno++)
2751     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2752       {
2753         SET_HARD_REG_BIT (mask, regno);
2754         spill_size += 8;
2755         n_spilled += 1;
2756       }
2757
2758   /* Now come all special registers that might get saved in other
2759      general registers.  */
2760
2761   if (frame_pointer_needed)
2762     {
2763       current_frame_info.r[reg_fp] = find_gr_spill (reg_fp, 1);
2764       /* If we did not get a register, then we take LOC79.  This is guaranteed
2765          to be free, even if regs_ever_live is already set, because this is
2766          HARD_FRAME_POINTER_REGNUM.  This requires incrementing n_local_regs,
2767          as we don't count loc79 above.  */
2768       if (current_frame_info.r[reg_fp] == 0)
2769         {
2770           current_frame_info.r[reg_fp] = LOC_REG (79);
2771           current_frame_info.n_local_regs = LOC_REG (79) - LOC_REG (0) + 1;
2772         }
2773     }
2774
2775   if (! crtl->is_leaf)
2776     {
2777       /* Emit a save of BR0 if we call other functions.  Do this even
2778          if this function doesn't return, as EH depends on this to be
2779          able to unwind the stack.  */
2780       SET_HARD_REG_BIT (mask, BR_REG (0));
2781
2782       current_frame_info.r[reg_save_b0] = find_gr_spill (reg_save_b0, 1);
2783       if (current_frame_info.r[reg_save_b0] == 0)
2784         {
2785           extra_spill_size += 8;
2786           n_spilled += 1;
2787         }
2788
2789       /* Similarly for ar.pfs.  */
2790       SET_HARD_REG_BIT (mask, AR_PFS_REGNUM);
2791       current_frame_info.r[reg_save_ar_pfs] = find_gr_spill (reg_save_ar_pfs, 1);
2792       if (current_frame_info.r[reg_save_ar_pfs] == 0)
2793         {
2794           extra_spill_size += 8;
2795           n_spilled += 1;
2796         }
2797
2798       /* Similarly for gp.  Note that if we're calling setjmp, the stacked
2799          registers are clobbered, so we fall back to the stack.  */
2800       current_frame_info.r[reg_save_gp]
2801         = (cfun->calls_setjmp ? 0 : find_gr_spill (reg_save_gp, 1));
2802       if (current_frame_info.r[reg_save_gp] == 0)
2803         {
2804           SET_HARD_REG_BIT (mask, GR_REG (1));
2805           spill_size += 8;
2806           n_spilled += 1;
2807         }
2808     }
2809   else
2810     {
2811       if (df_regs_ever_live_p (BR_REG (0)) && ! call_used_regs[BR_REG (0)])
2812         {
2813           SET_HARD_REG_BIT (mask, BR_REG (0));
2814           extra_spill_size += 8;
2815           n_spilled += 1;
2816         }
2817
2818       if (df_regs_ever_live_p (AR_PFS_REGNUM))
2819         {
2820           SET_HARD_REG_BIT (mask, AR_PFS_REGNUM);
2821           current_frame_info.r[reg_save_ar_pfs]
2822             = find_gr_spill (reg_save_ar_pfs, 1);
2823           if (current_frame_info.r[reg_save_ar_pfs] == 0)
2824             {
2825               extra_spill_size += 8;
2826               n_spilled += 1;
2827             }
2828         }
2829     }
2830
2831   /* Unwind descriptor hackery: things are most efficient if we allocate
2832      consecutive GR save registers for RP, PFS, FP in that order. However,
2833      it is absolutely critical that FP get the only hard register that's
2834      guaranteed to be free, so we allocated it first.  If all three did
2835      happen to be allocated hard regs, and are consecutive, rearrange them
2836      into the preferred order now.
2837
2838      If we have already emitted code for any of those registers,
2839      then it's already too late to change.  */
2840   min_regno = MIN (current_frame_info.r[reg_fp],
2841                    MIN (current_frame_info.r[reg_save_b0],
2842                         current_frame_info.r[reg_save_ar_pfs]));
2843   max_regno = MAX (current_frame_info.r[reg_fp],
2844                    MAX (current_frame_info.r[reg_save_b0],
2845                         current_frame_info.r[reg_save_ar_pfs]));
2846   if (min_regno > 0
2847       && min_regno + 2 == max_regno
2848       && (current_frame_info.r[reg_fp] == min_regno + 1
2849           || current_frame_info.r[reg_save_b0] == min_regno + 1
2850           || current_frame_info.r[reg_save_ar_pfs] == min_regno + 1)
2851       && (emitted_frame_related_regs[reg_save_b0] == 0
2852           || emitted_frame_related_regs[reg_save_b0] == min_regno)
2853       && (emitted_frame_related_regs[reg_save_ar_pfs] == 0
2854           || emitted_frame_related_regs[reg_save_ar_pfs] == min_regno + 1)
2855       && (emitted_frame_related_regs[reg_fp] == 0
2856           || emitted_frame_related_regs[reg_fp] == min_regno + 2))
2857     {
2858       current_frame_info.r[reg_save_b0] = min_regno;
2859       current_frame_info.r[reg_save_ar_pfs] = min_regno + 1;
2860       current_frame_info.r[reg_fp] = min_regno + 2;
2861     }
2862
2863   /* See if we need to store the predicate register block.  */
2864   for (regno = PR_REG (0); regno <= PR_REG (63); regno++)
2865     if (df_regs_ever_live_p (regno) && ! call_used_regs[regno])
2866       break;
2867   if (regno <= PR_REG (63))
2868     {
2869       SET_HARD_REG_BIT (mask, PR_REG (0));
2870       current_frame_info.r[reg_save_pr] = find_gr_spill (reg_save_pr, 1);
2871       if (current_frame_info.r[reg_save_pr] == 0)
2872         {
2873           extra_spill_size += 8;
2874           n_spilled += 1;
2875         }
2876
2877       /* ??? Mark them all as used so that register renaming and such
2878          are free to use them.  */
2879       for (regno = PR_REG (0); regno <= PR_REG (63); regno++)
2880         df_set_regs_ever_live (regno, true);
2881     }
2882
2883   /* If we're forced to use st8.spill, we're forced to save and restore
2884      ar.unat as well.  The check for existing liveness allows inline asm
2885      to touch ar.unat.  */
2886   if (spilled_gr_p || cfun->machine->n_varargs
2887       || df_regs_ever_live_p (AR_UNAT_REGNUM))
2888     {
2889       df_set_regs_ever_live (AR_UNAT_REGNUM, true);
2890       SET_HARD_REG_BIT (mask, AR_UNAT_REGNUM);
2891       current_frame_info.r[reg_save_ar_unat]
2892         = find_gr_spill (reg_save_ar_unat, spill_size == 0);
2893       if (current_frame_info.r[reg_save_ar_unat] == 0)
2894         {
2895           extra_spill_size += 8;
2896           n_spilled += 1;
2897         }
2898     }
2899
2900   if (df_regs_ever_live_p (AR_LC_REGNUM))
2901     {
2902       SET_HARD_REG_BIT (mask, AR_LC_REGNUM);
2903       current_frame_info.r[reg_save_ar_lc]
2904         = find_gr_spill (reg_save_ar_lc, spill_size == 0);
2905       if (current_frame_info.r[reg_save_ar_lc] == 0)
2906         {
2907           extra_spill_size += 8;
2908           n_spilled += 1;
2909         }
2910     }
2911
2912   /* If we have an odd number of words of pretend arguments written to
2913      the stack, then the FR save area will be unaligned.  We round the
2914      size of this area up to keep things 16 byte aligned.  */
2915   if (spilled_fr_p)
2916     pretend_args_size = IA64_STACK_ALIGN (crtl->args.pretend_args_size);
2917   else
2918     pretend_args_size = crtl->args.pretend_args_size;
2919
2920   total_size = (spill_size + extra_spill_size + size + pretend_args_size
2921                 + crtl->outgoing_args_size);
2922   total_size = IA64_STACK_ALIGN (total_size);
2923
2924   /* We always use the 16-byte scratch area provided by the caller, but
2925      if we are a leaf function, there's no one to which we need to provide
2926      a scratch area.  However, if the function allocates dynamic stack space,
2927      the dynamic offset is computed early and contains STACK_POINTER_OFFSET,
2928      so we need to cope.  */
2929   if (crtl->is_leaf && !cfun->calls_alloca)
2930     total_size = MAX (0, total_size - 16);
2931
2932   current_frame_info.total_size = total_size;
2933   current_frame_info.spill_cfa_off = pretend_args_size - 16;
2934   current_frame_info.spill_size = spill_size;
2935   current_frame_info.extra_spill_size = extra_spill_size;
2936   COPY_HARD_REG_SET (current_frame_info.mask, mask);
2937   current_frame_info.n_spilled = n_spilled;
2938   current_frame_info.initialized = reload_completed;
2939 }
2940
2941 /* Worker function for TARGET_CAN_ELIMINATE.  */
2942
2943 bool
2944 ia64_can_eliminate (const int from ATTRIBUTE_UNUSED, const int to)
2945 {
2946   return (to == BR_REG (0) ? crtl->is_leaf : true);
2947 }
2948
2949 /* Compute the initial difference between the specified pair of registers.  */
2950
2951 HOST_WIDE_INT
2952 ia64_initial_elimination_offset (int from, int to)
2953 {
2954   HOST_WIDE_INT offset;
2955
2956   ia64_compute_frame_size (get_frame_size ());
2957   switch (from)
2958     {
2959     case FRAME_POINTER_REGNUM:
2960       switch (to)
2961         {
2962         case HARD_FRAME_POINTER_REGNUM:
2963           offset = -current_frame_info.total_size;
2964           if (!crtl->is_leaf || cfun->calls_alloca)
2965             offset += 16 + crtl->outgoing_args_size;
2966           break;
2967
2968         case STACK_POINTER_REGNUM:
2969           offset = 0;
2970           if (!crtl->is_leaf || cfun->calls_alloca)
2971             offset += 16 + crtl->outgoing_args_size;
2972           break;
2973
2974         default:
2975           gcc_unreachable ();
2976         }
2977       break;
2978
2979     case ARG_POINTER_REGNUM:
2980       /* Arguments start above the 16 byte save area, unless stdarg
2981          in which case we store through the 16 byte save area.  */
2982       switch (to)
2983         {
2984         case HARD_FRAME_POINTER_REGNUM:
2985           offset = 16 - crtl->args.pretend_args_size;
2986           break;
2987
2988         case STACK_POINTER_REGNUM:
2989           offset = (current_frame_info.total_size
2990                     + 16 - crtl->args.pretend_args_size);
2991           break;
2992
2993         default:
2994           gcc_unreachable ();
2995         }
2996       break;
2997
2998     default:
2999       gcc_unreachable ();
3000     }
3001
3002   return offset;
3003 }
3004
3005 /* If there are more than a trivial number of register spills, we use
3006    two interleaved iterators so that we can get two memory references
3007    per insn group.
3008
3009    In order to simplify things in the prologue and epilogue expanders,
3010    we use helper functions to fix up the memory references after the
3011    fact with the appropriate offsets to a POST_MODIFY memory mode.
3012    The following data structure tracks the state of the two iterators
3013    while insns are being emitted.  */
3014
3015 struct spill_fill_data
3016 {
3017   rtx_insn *init_after;         /* point at which to emit initializations */
3018   rtx init_reg[2];              /* initial base register */
3019   rtx iter_reg[2];              /* the iterator registers */
3020   rtx *prev_addr[2];            /* address of last memory use */
3021   rtx_insn *prev_insn[2];       /* the insn corresponding to prev_addr */
3022   HOST_WIDE_INT prev_off[2];    /* last offset */
3023   int n_iter;                   /* number of iterators in use */
3024   int next_iter;                /* next iterator to use */
3025   unsigned int save_gr_used_mask;
3026 };
3027
3028 static struct spill_fill_data spill_fill_data;
3029
3030 static void
3031 setup_spill_pointers (int n_spills, rtx init_reg, HOST_WIDE_INT cfa_off)
3032 {
3033   int i;
3034
3035   spill_fill_data.init_after = get_last_insn ();
3036   spill_fill_data.init_reg[0] = init_reg;
3037   spill_fill_data.init_reg[1] = init_reg;
3038   spill_fill_data.prev_addr[0] = NULL;
3039   spill_fill_data.prev_addr[1] = NULL;
3040   spill_fill_data.prev_insn[0] = NULL;
3041   spill_fill_data.prev_insn[1] = NULL;
3042   spill_fill_data.prev_off[0] = cfa_off;
3043   spill_fill_data.prev_off[1] = cfa_off;
3044   spill_fill_data.next_iter = 0;
3045   spill_fill_data.save_gr_used_mask = current_frame_info.gr_used_mask;
3046
3047   spill_fill_data.n_iter = 1 + (n_spills > 2);
3048   for (i = 0; i < spill_fill_data.n_iter; ++i)
3049     {
3050       int regno = next_scratch_gr_reg ();
3051       spill_fill_data.iter_reg[i] = gen_rtx_REG (DImode, regno);
3052       current_frame_info.gr_used_mask |= 1 << regno;
3053     }
3054 }
3055
3056 static void
3057 finish_spill_pointers (void)
3058 {
3059   current_frame_info.gr_used_mask = spill_fill_data.save_gr_used_mask;
3060 }
3061
3062 static rtx
3063 spill_restore_mem (rtx reg, HOST_WIDE_INT cfa_off)
3064 {
3065   int iter = spill_fill_data.next_iter;
3066   HOST_WIDE_INT disp = spill_fill_data.prev_off[iter] - cfa_off;
3067   rtx disp_rtx = GEN_INT (disp);
3068   rtx mem;
3069
3070   if (spill_fill_data.prev_addr[iter])
3071     {
3072       if (satisfies_constraint_N (disp_rtx))
3073         {
3074           *spill_fill_data.prev_addr[iter]
3075             = gen_rtx_POST_MODIFY (DImode, spill_fill_data.iter_reg[iter],
3076                                    gen_rtx_PLUS (DImode,
3077                                                  spill_fill_data.iter_reg[iter],
3078                                                  disp_rtx));
3079           add_reg_note (spill_fill_data.prev_insn[iter],
3080                         REG_INC, spill_fill_data.iter_reg[iter]);
3081         }
3082       else
3083         {
3084           /* ??? Could use register post_modify for loads.  */
3085           if (!satisfies_constraint_I (disp_rtx))
3086             {
3087               rtx tmp = gen_rtx_REG (DImode, next_scratch_gr_reg ());
3088               emit_move_insn (tmp, disp_rtx);
3089               disp_rtx = tmp;
3090             }
3091           emit_insn (gen_adddi3 (spill_fill_data.iter_reg[iter],
3092                                  spill_fill_data.iter_reg[iter], disp_rtx));
3093         }
3094     }
3095   /* Micro-optimization: if we've created a frame pointer, it's at
3096      CFA 0, which may allow the real iterator to be initialized lower,
3097      slightly increasing parallelism.  Also, if there are few saves
3098      it may eliminate the iterator entirely.  */
3099   else if (disp == 0
3100            && spill_fill_data.init_reg[iter] == stack_pointer_rtx
3101            && frame_pointer_needed)
3102     {
3103       mem = gen_rtx_MEM (GET_MODE (reg), hard_frame_pointer_rtx);
3104       set_mem_alias_set (mem, get_varargs_alias_set ());
3105       return mem;
3106     }
3107   else
3108     {
3109       rtx seq;
3110       rtx_insn *insn;
3111
3112       if (disp == 0)
3113         seq = gen_movdi (spill_fill_data.iter_reg[iter],
3114                          spill_fill_data.init_reg[iter]);
3115       else
3116         {
3117           start_sequence ();
3118
3119           if (!satisfies_constraint_I (disp_rtx))
3120             {
3121               rtx tmp = gen_rtx_REG (DImode, next_scratch_gr_reg ());
3122               emit_move_insn (tmp, disp_rtx);
3123               disp_rtx = tmp;
3124             }
3125
3126           emit_insn (gen_adddi3 (spill_fill_data.iter_reg[iter],
3127                                  spill_fill_data.init_reg[iter],
3128                                  disp_rtx));
3129
3130           seq = get_insns ();
3131           end_sequence ();
3132         }
3133
3134       /* Careful for being the first insn in a sequence.  */
3135       if (spill_fill_data.init_after)
3136         insn = emit_insn_after (seq, spill_fill_data.init_after);
3137       else
3138         {
3139           rtx_insn *first = get_insns ();
3140           if (first)
3141             insn = emit_insn_before (seq, first);
3142           else
3143             insn = emit_insn (seq);
3144         }
3145       spill_fill_data.init_after = insn;
3146     }
3147
3148   mem = gen_rtx_MEM (GET_MODE (reg), spill_fill_data.iter_reg[iter]);
3149
3150   /* ??? Not all of the spills are for varargs, but some of them are.
3151      The rest of the spills belong in an alias set of their own.  But
3152      it doesn't actually hurt to include them here.  */
3153   set_mem_alias_set (mem, get_varargs_alias_set ());
3154
3155   spill_fill_data.prev_addr[iter] = &XEXP (mem, 0);
3156   spill_fill_data.prev_off[iter] = cfa_off;
3157
3158   if (++iter >= spill_fill_data.n_iter)
3159     iter = 0;
3160   spill_fill_data.next_iter = iter;
3161
3162   return mem;
3163 }
3164
3165 static void
3166 do_spill (rtx (*move_fn) (rtx, rtx, rtx), rtx reg, HOST_WIDE_INT cfa_off,
3167           rtx frame_reg)
3168 {
3169   int iter = spill_fill_data.next_iter;
3170   rtx mem;
3171   rtx_insn *insn;
3172
3173   mem = spill_restore_mem (reg, cfa_off);
3174   insn = emit_insn ((*move_fn) (mem, reg, GEN_INT (cfa_off)));
3175   spill_fill_data.prev_insn[iter] = insn;
3176
3177   if (frame_reg)
3178     {
3179       rtx base;
3180       HOST_WIDE_INT off;
3181
3182       RTX_FRAME_RELATED_P (insn) = 1;
3183
3184       /* Don't even pretend that the unwind code can intuit its way
3185          through a pair of interleaved post_modify iterators.  Just
3186          provide the correct answer.  */
3187
3188       if (frame_pointer_needed)
3189         {
3190           base = hard_frame_pointer_rtx;
3191           off = - cfa_off;
3192         }
3193       else
3194         {
3195           base = stack_pointer_rtx;
3196           off = current_frame_info.total_size - cfa_off;
3197         }
3198
3199       add_reg_note (insn, REG_CFA_OFFSET,
3200                     gen_rtx_SET (VOIDmode,
3201                                  gen_rtx_MEM (GET_MODE (reg),
3202                                               plus_constant (Pmode,
3203                                                              base, off)),
3204                                  frame_reg));
3205     }
3206 }
3207
3208 static void
3209 do_restore (rtx (*move_fn) (rtx, rtx, rtx), rtx reg, HOST_WIDE_INT cfa_off)
3210 {
3211   int iter = spill_fill_data.next_iter;
3212   rtx_insn *insn;
3213
3214   insn = emit_insn ((*move_fn) (reg, spill_restore_mem (reg, cfa_off),
3215                                 GEN_INT (cfa_off)));
3216   spill_fill_data.prev_insn[iter] = insn;
3217 }
3218
3219 /* Wrapper functions that discards the CONST_INT spill offset.  These
3220    exist so that we can give gr_spill/gr_fill the offset they need and
3221    use a consistent function interface.  */
3222
3223 static rtx
3224 gen_movdi_x (rtx dest, rtx src, rtx offset ATTRIBUTE_UNUSED)
3225 {
3226   return gen_movdi (dest, src);
3227 }
3228
3229 static rtx
3230 gen_fr_spill_x (rtx dest, rtx src, rtx offset ATTRIBUTE_UNUSED)
3231 {
3232   return gen_fr_spill (dest, src);
3233 }
3234
3235 static rtx
3236 gen_fr_restore_x (rtx dest, rtx src, rtx offset ATTRIBUTE_UNUSED)
3237 {
3238   return gen_fr_restore (dest, src);
3239 }
3240
3241 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
3242
3243 /* See Table 6.2 of the IA-64 Software Developer Manual, Volume 2.  */
3244 #define BACKING_STORE_SIZE(N) ((N) > 0 ? ((N) + (N)/63 + 1) * 8 : 0)
3245
3246 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
3247    inclusive.  These are offsets from the current stack pointer.  BS_SIZE
3248    is the size of the backing store.  ??? This clobbers r2 and r3.  */
3249
3250 static void
3251 ia64_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size,
3252                              int bs_size)
3253 {
3254   rtx r2 = gen_rtx_REG (Pmode, GR_REG (2));
3255   rtx r3 = gen_rtx_REG (Pmode, GR_REG (3));
3256   rtx p6 = gen_rtx_REG (BImode, PR_REG (6));
3257
3258   /* On the IA-64 there is a second stack in memory, namely the Backing Store
3259      of the Register Stack Engine.  We also need to probe it after checking
3260      that the 2 stacks don't overlap.  */
3261   emit_insn (gen_bsp_value (r3));
3262   emit_move_insn (r2, GEN_INT (-(first + size)));
3263
3264   /* Compare current value of BSP and SP registers.  */
3265   emit_insn (gen_rtx_SET (VOIDmode, p6,
3266                           gen_rtx_fmt_ee (LTU, BImode,
3267                                           r3, stack_pointer_rtx)));
3268
3269   /* Compute the address of the probe for the Backing Store (which grows
3270      towards higher addresses).  We probe only at the first offset of
3271      the next page because some OS (eg Linux/ia64) only extend the
3272      backing store when this specific address is hit (but generate a SEGV
3273      on other address).  Page size is the worst case (4KB).  The reserve
3274      size is at least 4096 - (96 + 2) * 8 = 3312 bytes, which is enough.
3275      Also compute the address of the last probe for the memory stack
3276      (which grows towards lower addresses).  */
3277   emit_insn (gen_rtx_SET (VOIDmode, r3, plus_constant (Pmode, r3, 4095)));
3278   emit_insn (gen_rtx_SET (VOIDmode, r2,
3279                           gen_rtx_PLUS (Pmode, stack_pointer_rtx, r2)));
3280
3281   /* Compare them and raise SEGV if the former has topped the latter.  */
3282   emit_insn (gen_rtx_COND_EXEC (VOIDmode,
3283                                 gen_rtx_fmt_ee (NE, VOIDmode, p6, const0_rtx),
3284                                 gen_rtx_SET (VOIDmode, p6,
3285                                              gen_rtx_fmt_ee (GEU, BImode,
3286                                                              r3, r2))));
3287   emit_insn (gen_rtx_SET (VOIDmode,
3288                           gen_rtx_ZERO_EXTRACT (DImode, r3, GEN_INT (12),
3289                                                 const0_rtx),
3290                           const0_rtx));
3291   emit_insn (gen_rtx_COND_EXEC (VOIDmode,
3292                                 gen_rtx_fmt_ee (NE, VOIDmode, p6, const0_rtx),
3293                                 gen_rtx_TRAP_IF (VOIDmode, const1_rtx,
3294                                                  GEN_INT (11))));
3295
3296   /* Probe the Backing Store if necessary.  */
3297   if (bs_size > 0)
3298     emit_stack_probe (r3);
3299
3300   /* Probe the memory stack if necessary.  */
3301   if (size == 0)
3302     ;
3303
3304   /* See if we have a constant small number of probes to generate.  If so,
3305      that's the easy case.  */
3306   else if (size <= PROBE_INTERVAL)
3307     emit_stack_probe (r2);
3308
3309   /* The run-time loop is made up of 8 insns in the generic case while this
3310      compile-time loop is made up of 5+2*(n-2) insns for n # of intervals.  */
3311   else if (size <= 4 * PROBE_INTERVAL)
3312     {
3313       HOST_WIDE_INT i;
3314
3315       emit_move_insn (r2, GEN_INT (-(first + PROBE_INTERVAL)));
3316       emit_insn (gen_rtx_SET (VOIDmode, r2,
3317                               gen_rtx_PLUS (Pmode, stack_pointer_rtx, r2)));
3318       emit_stack_probe (r2);
3319
3320       /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
3321          it exceeds SIZE.  If only two probes are needed, this will not
3322          generate any code.  Then probe at FIRST + SIZE.  */
3323       for (i = 2 * PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
3324         {
3325           emit_insn (gen_rtx_SET (VOIDmode, r2,
3326                                   plus_constant (Pmode, r2, -PROBE_INTERVAL)));
3327           emit_stack_probe (r2);
3328         }
3329
3330       emit_insn (gen_rtx_SET (VOIDmode, r2,
3331                               plus_constant (Pmode, r2,
3332                                              (i - PROBE_INTERVAL) - size)));
3333       emit_stack_probe (r2);
3334     }
3335
3336   /* Otherwise, do the same as above, but in a loop.  Note that we must be
3337      extra careful with variables wrapping around because we might be at
3338      the very top (or the very bottom) of the address space and we have
3339      to be able to handle this case properly; in particular, we use an
3340      equality test for the loop condition.  */
3341   else
3342     {
3343       HOST_WIDE_INT rounded_size;
3344
3345       emit_move_insn (r2, GEN_INT (-first));
3346
3347
3348       /* Step 1: round SIZE to the previous multiple of the interval.  */
3349
3350       rounded_size = size & -PROBE_INTERVAL;
3351
3352
3353       /* Step 2: compute initial and final value of the loop counter.  */
3354
3355       /* TEST_ADDR = SP + FIRST.  */
3356       emit_insn (gen_rtx_SET (VOIDmode, r2,
3357                               gen_rtx_PLUS (Pmode, stack_pointer_rtx, r2)));
3358
3359       /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE.  */
3360       if (rounded_size > (1 << 21))
3361         {
3362           emit_move_insn (r3, GEN_INT (-rounded_size));
3363           emit_insn (gen_rtx_SET (VOIDmode, r3, gen_rtx_PLUS (Pmode, r2, r3)));
3364         }
3365       else
3366         emit_insn (gen_rtx_SET (VOIDmode, r3,
3367                                 gen_rtx_PLUS (Pmode, r2,
3368                                               GEN_INT (-rounded_size))));
3369
3370
3371       /* Step 3: the loop
3372
3373          while (TEST_ADDR != LAST_ADDR)
3374            {
3375              TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
3376              probe at TEST_ADDR
3377            }
3378
3379          probes at FIRST + N * PROBE_INTERVAL for values of N from 1
3380          until it is equal to ROUNDED_SIZE.  */
3381
3382       emit_insn (gen_probe_stack_range (r2, r2, r3));
3383
3384
3385       /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
3386          that SIZE is equal to ROUNDED_SIZE.  */
3387
3388       /* TEMP = SIZE - ROUNDED_SIZE.  */
3389       if (size != rounded_size)
3390         {
3391           emit_insn (gen_rtx_SET (VOIDmode, r2,
3392                                   plus_constant (Pmode, r2,
3393                                                  rounded_size - size)));
3394           emit_stack_probe (r2);
3395         }
3396     }
3397
3398   /* Make sure nothing is scheduled before we are done.  */
3399   emit_insn (gen_blockage ());
3400 }
3401
3402 /* Probe a range of stack addresses from REG1 to REG2 inclusive.  These are
3403    absolute addresses.  */
3404
3405 const char *
3406 output_probe_stack_range (rtx reg1, rtx reg2)
3407 {
3408   static int labelno = 0;
3409   char loop_lab[32], end_lab[32];
3410   rtx xops[3];
3411
3412   ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno);
3413   ASM_GENERATE_INTERNAL_LABEL (end_lab, "LPSRE", labelno++);
3414
3415   ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab);
3416
3417   /* Jump to END_LAB if TEST_ADDR == LAST_ADDR.  */
3418   xops[0] = reg1;
3419   xops[1] = reg2;
3420   xops[2] = gen_rtx_REG (BImode, PR_REG (6));
3421   output_asm_insn ("cmp.eq %2, %I2 = %0, %1", xops);
3422   fprintf (asm_out_file, "\t(%s) br.cond.dpnt ", reg_names [REGNO (xops[2])]);
3423   assemble_name_raw (asm_out_file, end_lab);
3424   fputc ('\n', asm_out_file);
3425
3426   /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL.  */
3427   xops[1] = GEN_INT (-PROBE_INTERVAL);
3428   output_asm_insn ("addl %0 = %1, %0", xops);
3429   fputs ("\t;;\n", asm_out_file);
3430
3431   /* Probe at TEST_ADDR and branch.  */
3432   output_asm_insn ("probe.w.fault %0, 0", xops);
3433   fprintf (asm_out_file, "\tbr ");
3434   assemble_name_raw (asm_out_file, loop_lab);
3435   fputc ('\n', asm_out_file);
3436
3437   ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, end_lab);
3438
3439   return "";
3440 }
3441
3442 /* Called after register allocation to add any instructions needed for the
3443    prologue.  Using a prologue insn is favored compared to putting all of the
3444    instructions in output_function_prologue(), since it allows the scheduler
3445    to intermix instructions with the saves of the caller saved registers.  In
3446    some cases, it might be necessary to emit a barrier instruction as the last
3447    insn to prevent such scheduling.
3448
3449    Also any insns generated here should have RTX_FRAME_RELATED_P(insn) = 1
3450    so that the debug info generation code can handle them properly.
3451
3452    The register save area is laid out like so:
3453    cfa+16
3454         [ varargs spill area ]
3455         [ fr register spill area ]
3456         [ br register spill area ]
3457         [ ar register spill area ]
3458         [ pr register spill area ]
3459         [ gr register spill area ] */
3460
3461 /* ??? Get inefficient code when the frame size is larger than can fit in an
3462    adds instruction.  */
3463
3464 void
3465 ia64_expand_prologue (void)
3466 {
3467   rtx_insn *insn;
3468   rtx ar_pfs_save_reg, ar_unat_save_reg;
3469   int i, epilogue_p, regno, alt_regno, cfa_off, n_varargs;
3470   rtx reg, alt_reg;
3471
3472   ia64_compute_frame_size (get_frame_size ());
3473   last_scratch_gr_reg = 15;
3474
3475   if (flag_stack_usage_info)
3476     current_function_static_stack_size = current_frame_info.total_size;
3477
3478   if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
3479     {
3480       HOST_WIDE_INT size = current_frame_info.total_size;
3481       int bs_size = BACKING_STORE_SIZE (current_frame_info.n_input_regs
3482                                           + current_frame_info.n_local_regs);
3483
3484       if (crtl->is_leaf && !cfun->calls_alloca)
3485         {
3486           if (size > PROBE_INTERVAL && size > STACK_CHECK_PROTECT)
3487             ia64_emit_probe_stack_range (STACK_CHECK_PROTECT,
3488                                          size - STACK_CHECK_PROTECT,
3489                                          bs_size);
3490           else if (size + bs_size > STACK_CHECK_PROTECT)
3491             ia64_emit_probe_stack_range (STACK_CHECK_PROTECT, 0, bs_size);
3492         }
3493       else if (size + bs_size > 0)
3494         ia64_emit_probe_stack_range (STACK_CHECK_PROTECT, size, bs_size);
3495     }
3496
3497   if (dump_file)
3498     {
3499       fprintf (dump_file, "ia64 frame related registers "
3500                "recorded in current_frame_info.r[]:\n");
3501 #define PRINTREG(a) if (current_frame_info.r[a]) \
3502         fprintf(dump_file, "%s = %d\n", #a, current_frame_info.r[a])
3503       PRINTREG(reg_fp);
3504       PRINTREG(reg_save_b0);
3505       PRINTREG(reg_save_pr);
3506       PRINTREG(reg_save_ar_pfs);
3507       PRINTREG(reg_save_ar_unat);
3508       PRINTREG(reg_save_ar_lc);
3509       PRINTREG(reg_save_gp);
3510 #undef PRINTREG
3511     }
3512
3513   /* If there is no epilogue, then we don't need some prologue insns.
3514      We need to avoid emitting the dead prologue insns, because flow
3515      will complain about them.  */
3516   if (optimize)
3517     {
3518       edge e;
3519       edge_iterator ei;
3520
3521       FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
3522         if ((e->flags & EDGE_FAKE) == 0
3523             && (e->flags & EDGE_FALLTHRU) != 0)
3524           break;
3525       epilogue_p = (e != NULL);
3526     }
3527   else
3528     epilogue_p = 1;
3529
3530   /* Set the local, input, and output register names.  We need to do this
3531      for GNU libc, which creates crti.S/crtn.S by splitting initfini.c in
3532      half.  If we use in/loc/out register names, then we get assembler errors
3533      in crtn.S because there is no alloc insn or regstk directive in there.  */
3534   if (! TARGET_REG_NAMES)
3535     {
3536       int inputs = current_frame_info.n_input_regs;
3537       int locals = current_frame_info.n_local_regs;
3538       int outputs = current_frame_info.n_output_regs;
3539
3540       for (i = 0; i < inputs; i++)
3541         reg_names[IN_REG (i)] = ia64_reg_numbers[i];
3542       for (i = 0; i < locals; i++)
3543         reg_names[LOC_REG (i)] = ia64_reg_numbers[inputs + i];
3544       for (i = 0; i < outputs; i++)
3545         reg_names[OUT_REG (i)] = ia64_reg_numbers[inputs + locals + i];
3546     }
3547
3548   /* Set the frame pointer register name.  The regnum is logically loc79,
3549      but of course we'll not have allocated that many locals.  Rather than
3550      worrying about renumbering the existing rtxs, we adjust the name.  */
3551   /* ??? This code means that we can never use one local register when
3552      there is a frame pointer.  loc79 gets wasted in this case, as it is
3553      renamed to a register that will never be used.  See also the try_locals
3554      code in find_gr_spill.  */
3555   if (current_frame_info.r[reg_fp])
3556     {
3557       const char *tmp = reg_names[HARD_FRAME_POINTER_REGNUM];
3558       reg_names[HARD_FRAME_POINTER_REGNUM]
3559         = reg_names[current_frame_info.r[reg_fp]];
3560       reg_names[current_frame_info.r[reg_fp]] = tmp;
3561     }
3562
3563   /* We don't need an alloc instruction if we've used no outputs or locals.  */
3564   if (current_frame_info.n_local_regs == 0
3565       && current_frame_info.n_output_regs == 0
3566       && current_frame_info.n_input_regs <= crtl->args.info.int_regs
3567       && !TEST_HARD_REG_BIT (current_frame_info.mask, AR_PFS_REGNUM))
3568     {
3569       /* If there is no alloc, but there are input registers used, then we
3570          need a .regstk directive.  */
3571       current_frame_info.need_regstk = (TARGET_REG_NAMES != 0);
3572       ar_pfs_save_reg = NULL_RTX;
3573     }
3574   else
3575     {
3576       current_frame_info.need_regstk = 0;
3577
3578       if (current_frame_info.r[reg_save_ar_pfs])
3579         {
3580           regno = current_frame_info.r[reg_save_ar_pfs];
3581           reg_emitted (reg_save_ar_pfs);
3582         }
3583       else
3584         regno = next_scratch_gr_reg ();
3585       ar_pfs_save_reg = gen_rtx_REG (DImode, regno);
3586
3587       insn = emit_insn (gen_alloc (ar_pfs_save_reg,
3588                                    GEN_INT (current_frame_info.n_input_regs),
3589                                    GEN_INT (current_frame_info.n_local_regs),
3590                                    GEN_INT (current_frame_info.n_output_regs),
3591                                    GEN_INT (current_frame_info.n_rotate_regs)));
3592       if (current_frame_info.r[reg_save_ar_pfs])
3593         {
3594           RTX_FRAME_RELATED_P (insn) = 1;
3595           add_reg_note (insn, REG_CFA_REGISTER,
3596                         gen_rtx_SET (VOIDmode,
3597                                      ar_pfs_save_reg,
3598                                      gen_rtx_REG (DImode, AR_PFS_REGNUM)));
3599         }
3600     }
3601
3602   /* Set up frame pointer, stack pointer, and spill iterators.  */
3603
3604   n_varargs = cfun->machine->n_varargs;
3605   setup_spill_pointers (current_frame_info.n_spilled + n_varargs,
3606                         stack_pointer_rtx, 0);
3607
3608   if (frame_pointer_needed)
3609     {
3610       insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
3611       RTX_FRAME_RELATED_P (insn) = 1;
3612
3613       /* Force the unwind info to recognize this as defining a new CFA,
3614          rather than some temp register setup.  */
3615       add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL_RTX);
3616     }
3617
3618   if (current_frame_info.total_size != 0)
3619     {
3620       rtx frame_size_rtx = GEN_INT (- current_frame_info.total_size);
3621       rtx offset;
3622
3623       if (satisfies_constraint_I (frame_size_rtx))
3624         offset = frame_size_rtx;
3625       else
3626         {
3627           regno = next_scratch_gr_reg ();
3628           offset = gen_rtx_REG (DImode, regno);
3629           emit_move_insn (offset, frame_size_rtx);
3630         }
3631
3632       insn = emit_insn (gen_adddi3 (stack_pointer_rtx,
3633                                     stack_pointer_rtx, offset));
3634
3635       if (! frame_pointer_needed)
3636         {
3637           RTX_FRAME_RELATED_P (insn) = 1;
3638           add_reg_note (insn, REG_CFA_ADJUST_CFA,
3639                         gen_rtx_SET (VOIDmode,
3640                                      stack_pointer_rtx,
3641                                      gen_rtx_PLUS (DImode,
3642                                                    stack_pointer_rtx,
3643                                                    frame_size_rtx)));
3644         }
3645
3646       /* ??? At this point we must generate a magic insn that appears to
3647          modify the stack pointer, the frame pointer, and all spill
3648          iterators.  This would allow the most scheduling freedom.  For
3649          now, just hard stop.  */
3650       emit_insn (gen_blockage ());
3651     }
3652
3653   /* Must copy out ar.unat before doing any integer spills.  */
3654   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM))
3655     {
3656       if (current_frame_info.r[reg_save_ar_unat])
3657         {
3658           ar_unat_save_reg
3659             = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_unat]);
3660           reg_emitted (reg_save_ar_unat);
3661         }
3662       else
3663         {
3664           alt_regno = next_scratch_gr_reg ();
3665           ar_unat_save_reg = gen_rtx_REG (DImode, alt_regno);
3666           current_frame_info.gr_used_mask |= 1 << alt_regno;
3667         }
3668
3669       reg = gen_rtx_REG (DImode, AR_UNAT_REGNUM);
3670       insn = emit_move_insn (ar_unat_save_reg, reg);
3671       if (current_frame_info.r[reg_save_ar_unat])
3672         {
3673           RTX_FRAME_RELATED_P (insn) = 1;
3674           add_reg_note (insn, REG_CFA_REGISTER, NULL_RTX);
3675         }
3676
3677       /* Even if we're not going to generate an epilogue, we still
3678          need to save the register so that EH works.  */
3679       if (! epilogue_p && current_frame_info.r[reg_save_ar_unat])
3680         emit_insn (gen_prologue_use (ar_unat_save_reg));
3681     }
3682   else
3683     ar_unat_save_reg = NULL_RTX;
3684
3685   /* Spill all varargs registers.  Do this before spilling any GR registers,
3686      since we want the UNAT bits for the GR registers to override the UNAT
3687      bits from varargs, which we don't care about.  */
3688
3689   cfa_off = -16;
3690   for (regno = GR_ARG_FIRST + 7; n_varargs > 0; --n_varargs, --regno)
3691     {
3692       reg = gen_rtx_REG (DImode, regno);
3693       do_spill (gen_gr_spill, reg, cfa_off += 8, NULL_RTX);
3694     }
3695
3696   /* Locate the bottom of the register save area.  */
3697   cfa_off = (current_frame_info.spill_cfa_off
3698              + current_frame_info.spill_size
3699              + current_frame_info.extra_spill_size);
3700
3701   /* Save the predicate register block either in a register or in memory.  */
3702   if (TEST_HARD_REG_BIT (current_frame_info.mask, PR_REG (0)))
3703     {
3704       reg = gen_rtx_REG (DImode, PR_REG (0));
3705       if (current_frame_info.r[reg_save_pr] != 0)
3706         {
3707           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_pr]);
3708           reg_emitted (reg_save_pr);
3709           insn = emit_move_insn (alt_reg, reg);
3710
3711           /* ??? Denote pr spill/fill by a DImode move that modifies all
3712              64 hard registers.  */
3713           RTX_FRAME_RELATED_P (insn) = 1;
3714           add_reg_note (insn, REG_CFA_REGISTER, NULL_RTX);
3715
3716           /* Even if we're not going to generate an epilogue, we still
3717              need to save the register so that EH works.  */
3718           if (! epilogue_p)
3719             emit_insn (gen_prologue_use (alt_reg));
3720         }
3721       else
3722         {
3723           alt_regno = next_scratch_gr_reg ();
3724           alt_reg = gen_rtx_REG (DImode, alt_regno);
3725           insn = emit_move_insn (alt_reg, reg);
3726           do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3727           cfa_off -= 8;
3728         }
3729     }
3730
3731   /* Handle AR regs in numerical order.  All of them get special handling.  */
3732   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM)
3733       && current_frame_info.r[reg_save_ar_unat] == 0)
3734     {
3735       reg = gen_rtx_REG (DImode, AR_UNAT_REGNUM);
3736       do_spill (gen_movdi_x, ar_unat_save_reg, cfa_off, reg);
3737       cfa_off -= 8;
3738     }
3739
3740   /* The alloc insn already copied ar.pfs into a general register.  The
3741      only thing we have to do now is copy that register to a stack slot
3742      if we'd not allocated a local register for the job.  */
3743   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_PFS_REGNUM)
3744       && current_frame_info.r[reg_save_ar_pfs] == 0)
3745     {
3746       reg = gen_rtx_REG (DImode, AR_PFS_REGNUM);
3747       do_spill (gen_movdi_x, ar_pfs_save_reg, cfa_off, reg);
3748       cfa_off -= 8;
3749     }
3750
3751   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_LC_REGNUM))
3752     {
3753       reg = gen_rtx_REG (DImode, AR_LC_REGNUM);
3754       if (current_frame_info.r[reg_save_ar_lc] != 0)
3755         {
3756           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_lc]);
3757           reg_emitted (reg_save_ar_lc);
3758           insn = emit_move_insn (alt_reg, reg);
3759           RTX_FRAME_RELATED_P (insn) = 1;
3760           add_reg_note (insn, REG_CFA_REGISTER, NULL_RTX);
3761
3762           /* Even if we're not going to generate an epilogue, we still
3763              need to save the register so that EH works.  */
3764           if (! epilogue_p)
3765             emit_insn (gen_prologue_use (alt_reg));
3766         }
3767       else
3768         {
3769           alt_regno = next_scratch_gr_reg ();
3770           alt_reg = gen_rtx_REG (DImode, alt_regno);
3771           emit_move_insn (alt_reg, reg);
3772           do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3773           cfa_off -= 8;
3774         }
3775     }
3776
3777   /* Save the return pointer.  */
3778   if (TEST_HARD_REG_BIT (current_frame_info.mask, BR_REG (0)))
3779     {
3780       reg = gen_rtx_REG (DImode, BR_REG (0));
3781       if (current_frame_info.r[reg_save_b0] != 0)
3782         {
3783           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_b0]);
3784           reg_emitted (reg_save_b0);
3785           insn = emit_move_insn (alt_reg, reg);
3786           RTX_FRAME_RELATED_P (insn) = 1;
3787           add_reg_note (insn, REG_CFA_REGISTER,
3788                         gen_rtx_SET (VOIDmode, alt_reg, pc_rtx));
3789
3790           /* Even if we're not going to generate an epilogue, we still
3791              need to save the register so that EH works.  */
3792           if (! epilogue_p)
3793             emit_insn (gen_prologue_use (alt_reg));
3794         }
3795       else
3796         {
3797           alt_regno = next_scratch_gr_reg ();
3798           alt_reg = gen_rtx_REG (DImode, alt_regno);
3799           emit_move_insn (alt_reg, reg);
3800           do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3801           cfa_off -= 8;
3802         }
3803     }
3804
3805   if (current_frame_info.r[reg_save_gp])
3806     {
3807       reg_emitted (reg_save_gp);
3808       insn = emit_move_insn (gen_rtx_REG (DImode,
3809                                           current_frame_info.r[reg_save_gp]),
3810                              pic_offset_table_rtx);
3811     }
3812
3813   /* We should now be at the base of the gr/br/fr spill area.  */
3814   gcc_assert (cfa_off == (current_frame_info.spill_cfa_off
3815                           + current_frame_info.spill_size));
3816
3817   /* Spill all general registers.  */
3818   for (regno = GR_REG (1); regno <= GR_REG (31); ++regno)
3819     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
3820       {
3821         reg = gen_rtx_REG (DImode, regno);
3822         do_spill (gen_gr_spill, reg, cfa_off, reg);
3823         cfa_off -= 8;
3824       }
3825
3826   /* Spill the rest of the BR registers.  */
3827   for (regno = BR_REG (1); regno <= BR_REG (7); ++regno)
3828     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
3829       {
3830         alt_regno = next_scratch_gr_reg ();
3831         alt_reg = gen_rtx_REG (DImode, alt_regno);
3832         reg = gen_rtx_REG (DImode, regno);
3833         emit_move_insn (alt_reg, reg);
3834         do_spill (gen_movdi_x, alt_reg, cfa_off, reg);
3835         cfa_off -= 8;
3836       }
3837
3838   /* Align the frame and spill all FR registers.  */
3839   for (regno = FR_REG (2); regno <= FR_REG (127); ++regno)
3840     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
3841       {
3842         gcc_assert (!(cfa_off & 15));
3843         reg = gen_rtx_REG (XFmode, regno);
3844         do_spill (gen_fr_spill_x, reg, cfa_off, reg);
3845         cfa_off -= 16;
3846       }
3847
3848   gcc_assert (cfa_off == current_frame_info.spill_cfa_off);
3849
3850   finish_spill_pointers ();
3851 }
3852
3853 /* Output the textual info surrounding the prologue.  */
3854
3855 void
3856 ia64_start_function (FILE *file, const char *fnname,
3857                      tree decl ATTRIBUTE_UNUSED)
3858 {
3859 #if TARGET_ABI_OPEN_VMS
3860   vms_start_function (fnname);
3861 #endif
3862
3863   fputs ("\t.proc ", file);
3864   assemble_name (file, fnname);
3865   fputc ('\n', file);
3866   ASM_OUTPUT_LABEL (file, fnname);
3867 }
3868
3869 /* Called after register allocation to add any instructions needed for the
3870    epilogue.  Using an epilogue insn is favored compared to putting all of the
3871    instructions in output_function_prologue(), since it allows the scheduler
3872    to intermix instructions with the saves of the caller saved registers.  In
3873    some cases, it might be necessary to emit a barrier instruction as the last
3874    insn to prevent such scheduling.  */
3875
3876 void
3877 ia64_expand_epilogue (int sibcall_p)
3878 {
3879   rtx_insn *insn;
3880   rtx reg, alt_reg, ar_unat_save_reg;
3881   int regno, alt_regno, cfa_off;
3882
3883   ia64_compute_frame_size (get_frame_size ());
3884
3885   /* If there is a frame pointer, then we use it instead of the stack
3886      pointer, so that the stack pointer does not need to be valid when
3887      the epilogue starts.  See EXIT_IGNORE_STACK.  */
3888   if (frame_pointer_needed)
3889     setup_spill_pointers (current_frame_info.n_spilled,
3890                           hard_frame_pointer_rtx, 0);
3891   else
3892     setup_spill_pointers (current_frame_info.n_spilled, stack_pointer_rtx,
3893                           current_frame_info.total_size);
3894
3895   if (current_frame_info.total_size != 0)
3896     {
3897       /* ??? At this point we must generate a magic insn that appears to
3898          modify the spill iterators and the frame pointer.  This would
3899          allow the most scheduling freedom.  For now, just hard stop.  */
3900       emit_insn (gen_blockage ());
3901     }
3902
3903   /* Locate the bottom of the register save area.  */
3904   cfa_off = (current_frame_info.spill_cfa_off
3905              + current_frame_info.spill_size
3906              + current_frame_info.extra_spill_size);
3907
3908   /* Restore the predicate registers.  */
3909   if (TEST_HARD_REG_BIT (current_frame_info.mask, PR_REG (0)))
3910     {
3911       if (current_frame_info.r[reg_save_pr] != 0)
3912         {
3913           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_pr]);
3914           reg_emitted (reg_save_pr);
3915         }
3916       else
3917         {
3918           alt_regno = next_scratch_gr_reg ();
3919           alt_reg = gen_rtx_REG (DImode, alt_regno);
3920           do_restore (gen_movdi_x, alt_reg, cfa_off);
3921           cfa_off -= 8;
3922         }
3923       reg = gen_rtx_REG (DImode, PR_REG (0));
3924       emit_move_insn (reg, alt_reg);
3925     }
3926
3927   /* Restore the application registers.  */
3928
3929   /* Load the saved unat from the stack, but do not restore it until
3930      after the GRs have been restored.  */
3931   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM))
3932     {
3933       if (current_frame_info.r[reg_save_ar_unat] != 0)
3934         {
3935           ar_unat_save_reg
3936             = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_unat]);
3937           reg_emitted (reg_save_ar_unat);
3938         }
3939       else
3940         {
3941           alt_regno = next_scratch_gr_reg ();
3942           ar_unat_save_reg = gen_rtx_REG (DImode, alt_regno);
3943           current_frame_info.gr_used_mask |= 1 << alt_regno;
3944           do_restore (gen_movdi_x, ar_unat_save_reg, cfa_off);
3945           cfa_off -= 8;
3946         }
3947     }
3948   else
3949     ar_unat_save_reg = NULL_RTX;
3950
3951   if (current_frame_info.r[reg_save_ar_pfs] != 0)
3952     {
3953       reg_emitted (reg_save_ar_pfs);
3954       alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_pfs]);
3955       reg = gen_rtx_REG (DImode, AR_PFS_REGNUM);
3956       emit_move_insn (reg, alt_reg);
3957     }
3958   else if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_PFS_REGNUM))
3959     {
3960       alt_regno = next_scratch_gr_reg ();
3961       alt_reg = gen_rtx_REG (DImode, alt_regno);
3962       do_restore (gen_movdi_x, alt_reg, cfa_off);
3963       cfa_off -= 8;
3964       reg = gen_rtx_REG (DImode, AR_PFS_REGNUM);
3965       emit_move_insn (reg, alt_reg);
3966     }
3967
3968   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_LC_REGNUM))
3969     {
3970       if (current_frame_info.r[reg_save_ar_lc] != 0)
3971         {
3972           alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_ar_lc]);
3973           reg_emitted (reg_save_ar_lc);
3974         }
3975       else
3976         {
3977           alt_regno = next_scratch_gr_reg ();
3978           alt_reg = gen_rtx_REG (DImode, alt_regno);
3979           do_restore (gen_movdi_x, alt_reg, cfa_off);
3980           cfa_off -= 8;
3981         }
3982       reg = gen_rtx_REG (DImode, AR_LC_REGNUM);
3983       emit_move_insn (reg, alt_reg);
3984     }
3985
3986   /* Restore the return pointer.  */
3987   if (TEST_HARD_REG_BIT (current_frame_info.mask, BR_REG (0)))
3988     {
3989       if (current_frame_info.r[reg_save_b0] != 0)
3990         {
3991          alt_reg = gen_rtx_REG (DImode, current_frame_info.r[reg_save_b0]);
3992          reg_emitted (reg_save_b0);
3993         }
3994       else
3995         {
3996           alt_regno = next_scratch_gr_reg ();
3997           alt_reg = gen_rtx_REG (DImode, alt_regno);
3998           do_restore (gen_movdi_x, alt_reg, cfa_off);
3999           cfa_off -= 8;
4000         }
4001       reg = gen_rtx_REG (DImode, BR_REG (0));
4002       emit_move_insn (reg, alt_reg);
4003     }
4004
4005   /* We should now be at the base of the gr/br/fr spill area.  */
4006   gcc_assert (cfa_off == (current_frame_info.spill_cfa_off
4007                           + current_frame_info.spill_size));
4008
4009   /* The GP may be stored on the stack in the prologue, but it's
4010      never restored in the epilogue.  Skip the stack slot.  */
4011   if (TEST_HARD_REG_BIT (current_frame_info.mask, GR_REG (1)))
4012     cfa_off -= 8;
4013
4014   /* Restore all general registers.  */
4015   for (regno = GR_REG (2); regno <= GR_REG (31); ++regno)
4016     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4017       {
4018         reg = gen_rtx_REG (DImode, regno);
4019         do_restore (gen_gr_restore, reg, cfa_off);
4020         cfa_off -= 8;
4021       }
4022
4023   /* Restore the branch registers.  */
4024   for (regno = BR_REG (1); regno <= BR_REG (7); ++regno)
4025     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4026       {
4027         alt_regno = next_scratch_gr_reg ();
4028         alt_reg = gen_rtx_REG (DImode, alt_regno);
4029         do_restore (gen_movdi_x, alt_reg, cfa_off);
4030         cfa_off -= 8;
4031         reg = gen_rtx_REG (DImode, regno);
4032         emit_move_insn (reg, alt_reg);
4033       }
4034
4035   /* Restore floating point registers.  */
4036   for (regno = FR_REG (2); regno <= FR_REG (127); ++regno)
4037     if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4038       {
4039         gcc_assert (!(cfa_off & 15));
4040         reg = gen_rtx_REG (XFmode, regno);
4041         do_restore (gen_fr_restore_x, reg, cfa_off);
4042         cfa_off -= 16;
4043       }
4044
4045   /* Restore ar.unat for real.  */
4046   if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_UNAT_REGNUM))
4047     {
4048       reg = gen_rtx_REG (DImode, AR_UNAT_REGNUM);
4049       emit_move_insn (reg, ar_unat_save_reg);
4050     }
4051
4052   gcc_assert (cfa_off == current_frame_info.spill_cfa_off);
4053
4054   finish_spill_pointers ();
4055
4056   if (current_frame_info.total_size
4057       || cfun->machine->ia64_eh_epilogue_sp
4058       || frame_pointer_needed)
4059     {
4060       /* ??? At this point we must generate a magic insn that appears to
4061          modify the spill iterators, the stack pointer, and the frame
4062          pointer.  This would allow the most scheduling freedom.  For now,
4063          just hard stop.  */
4064       emit_insn (gen_blockage ());
4065     }
4066
4067   if (cfun->machine->ia64_eh_epilogue_sp)
4068     emit_move_insn (stack_pointer_rtx, cfun->machine->ia64_eh_epilogue_sp);
4069   else if (frame_pointer_needed)
4070     {
4071       insn = emit_move_insn (stack_pointer_rtx, hard_frame_pointer_rtx);
4072       RTX_FRAME_RELATED_P (insn) = 1;
4073       add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);
4074     }
4075   else if (current_frame_info.total_size)
4076     {
4077       rtx offset, frame_size_rtx;
4078
4079       frame_size_rtx = GEN_INT (current_frame_info.total_size);
4080       if (satisfies_constraint_I (frame_size_rtx))
4081         offset = frame_size_rtx;
4082       else
4083         {
4084           regno = next_scratch_gr_reg ();
4085           offset = gen_rtx_REG (DImode, regno);
4086           emit_move_insn (offset, frame_size_rtx);
4087         }
4088
4089       insn = emit_insn (gen_adddi3 (stack_pointer_rtx, stack_pointer_rtx,
4090                                     offset));
4091
4092       RTX_FRAME_RELATED_P (insn) = 1;
4093       add_reg_note (insn, REG_CFA_ADJUST_CFA,
4094                     gen_rtx_SET (VOIDmode,
4095                                  stack_pointer_rtx,
4096                                  gen_rtx_PLUS (DImode,
4097                                                stack_pointer_rtx,
4098                                                frame_size_rtx)));
4099     }
4100
4101   if (cfun->machine->ia64_eh_epilogue_bsp)
4102     emit_insn (gen_set_bsp (cfun->machine->ia64_eh_epilogue_bsp));
4103
4104   if (! sibcall_p)
4105     emit_jump_insn (gen_return_internal (gen_rtx_REG (DImode, BR_REG (0))));
4106   else
4107     {
4108       int fp = GR_REG (2);
4109       /* We need a throw away register here, r0 and r1 are reserved,
4110          so r2 is the first available call clobbered register.  If
4111          there was a frame_pointer register, we may have swapped the
4112          names of r2 and HARD_FRAME_POINTER_REGNUM, so we have to make
4113          sure we're using the string "r2" when emitting the register
4114          name for the assembler.  */
4115       if (current_frame_info.r[reg_fp]
4116           && current_frame_info.r[reg_fp] == GR_REG (2))
4117         fp = HARD_FRAME_POINTER_REGNUM;
4118
4119       /* We must emit an alloc to force the input registers to become output
4120          registers.  Otherwise, if the callee tries to pass its parameters
4121          through to another call without an intervening alloc, then these
4122          values get lost.  */
4123       /* ??? We don't need to preserve all input registers.  We only need to
4124          preserve those input registers used as arguments to the sibling call.
4125          It is unclear how to compute that number here.  */
4126       if (current_frame_info.n_input_regs != 0)
4127         {
4128           rtx n_inputs = GEN_INT (current_frame_info.n_input_regs);
4129
4130           insn = emit_insn (gen_alloc (gen_rtx_REG (DImode, fp),
4131                                 const0_rtx, const0_rtx,
4132                                 n_inputs, const0_rtx));
4133           RTX_FRAME_RELATED_P (insn) = 1;
4134
4135           /* ??? We need to mark the alloc as frame-related so that it gets
4136              passed into ia64_asm_unwind_emit for ia64-specific unwinding.
4137              But there's nothing dwarf2 related to be done wrt the register
4138              windows.  If we do nothing, dwarf2out will abort on the UNSPEC;
4139              the empty parallel means dwarf2out will not see anything.  */
4140           add_reg_note (insn, REG_FRAME_RELATED_EXPR,
4141                         gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (0)));
4142         }
4143     }
4144 }
4145
4146 /* Return 1 if br.ret can do all the work required to return from a
4147    function.  */
4148
4149 int
4150 ia64_direct_return (void)
4151 {
4152   if (reload_completed && ! frame_pointer_needed)
4153     {
4154       ia64_compute_frame_size (get_frame_size ());
4155
4156       return (current_frame_info.total_size == 0
4157               && current_frame_info.n_spilled == 0
4158               && current_frame_info.r[reg_save_b0] == 0
4159               && current_frame_info.r[reg_save_pr] == 0
4160               && current_frame_info.r[reg_save_ar_pfs] == 0
4161               && current_frame_info.r[reg_save_ar_unat] == 0
4162               && current_frame_info.r[reg_save_ar_lc] == 0);
4163     }
4164   return 0;
4165 }
4166
4167 /* Return the magic cookie that we use to hold the return address
4168    during early compilation.  */
4169
4170 rtx
4171 ia64_return_addr_rtx (HOST_WIDE_INT count, rtx frame ATTRIBUTE_UNUSED)
4172 {
4173   if (count != 0)
4174     return NULL;
4175   return gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx), UNSPEC_RET_ADDR);
4176 }
4177
4178 /* Split this value after reload, now that we know where the return
4179    address is saved.  */
4180
4181 void
4182 ia64_split_return_addr_rtx (rtx dest)
4183 {
4184   rtx src;
4185
4186   if (TEST_HARD_REG_BIT (current_frame_info.mask, BR_REG (0)))
4187     {
4188       if (current_frame_info.r[reg_save_b0] != 0)
4189         {
4190           src = gen_rtx_REG (DImode, current_frame_info.r[reg_save_b0]);
4191           reg_emitted (reg_save_b0);
4192         }
4193       else
4194         {
4195           HOST_WIDE_INT off;
4196           unsigned int regno;
4197           rtx off_r;
4198
4199           /* Compute offset from CFA for BR0.  */
4200           /* ??? Must be kept in sync with ia64_expand_prologue.  */
4201           off = (current_frame_info.spill_cfa_off
4202                  + current_frame_info.spill_size);
4203           for (regno = GR_REG (1); regno <= GR_REG (31); ++regno)
4204             if (TEST_HARD_REG_BIT (current_frame_info.mask, regno))
4205               off -= 8;
4206
4207           /* Convert CFA offset to a register based offset.  */
4208           if (frame_pointer_needed)
4209             src = hard_frame_pointer_rtx;
4210           else
4211             {
4212               src = stack_pointer_rtx;
4213               off += current_frame_info.total_size;
4214             }
4215
4216           /* Load address into scratch register.  */
4217           off_r = GEN_INT (off);
4218           if (satisfies_constraint_I (off_r))
4219             emit_insn (gen_adddi3 (dest, src, off_r));
4220           else
4221             {
4222               emit_move_insn (dest, off_r);
4223               emit_insn (gen_adddi3 (dest, src, dest));
4224             }
4225
4226           src = gen_rtx_MEM (Pmode, dest);
4227         }
4228     }
4229   else
4230     src = gen_rtx_REG (DImode, BR_REG (0));
4231
4232   emit_move_insn (dest, src);
4233 }
4234
4235 int
4236 ia64_hard_regno_rename_ok (int from, int to)
4237 {
4238   /* Don't clobber any of the registers we reserved for the prologue.  */
4239   unsigned int r;
4240
4241   for (r = reg_fp; r <= reg_save_ar_lc; r++)
4242     if (to == current_frame_info.r[r]
4243         || from == current_frame_info.r[r]
4244         || to == emitted_frame_related_regs[r]
4245         || from == emitted_frame_related_regs[r])
4246       return 0;
4247
4248   /* Don't use output registers outside the register frame.  */
4249   if (OUT_REGNO_P (to) && to >= OUT_REG (current_frame_info.n_output_regs))
4250     return 0;
4251
4252   /* Retain even/oddness on predicate register pairs.  */
4253   if (PR_REGNO_P (from) && PR_REGNO_P (to))
4254     return (from & 1) == (to & 1);
4255
4256   return 1;
4257 }
4258
4259 /* Target hook for assembling integer objects.  Handle word-sized
4260    aligned objects and detect the cases when @fptr is needed.  */
4261
4262 static bool
4263 ia64_assemble_integer (rtx x, unsigned int size, int aligned_p)
4264 {
4265   if (size == POINTER_SIZE / BITS_PER_UNIT
4266       && !(TARGET_NO_PIC || TARGET_AUTO_PIC)
4267       && GET_CODE (x) == SYMBOL_REF
4268       && SYMBOL_REF_FUNCTION_P (x))
4269     {
4270       static const char * const directive[2][2] = {
4271           /* 64-bit pointer */  /* 32-bit pointer */
4272         { "\tdata8.ua\t@fptr(", "\tdata4.ua\t@fptr("},  /* unaligned */
4273         { "\tdata8\t@fptr(",    "\tdata4\t@fptr("}      /* aligned */
4274       };
4275       fputs (directive[(aligned_p != 0)][POINTER_SIZE == 32], asm_out_file);
4276       output_addr_const (asm_out_file, x);
4277       fputs (")\n", asm_out_file);
4278       return true;
4279     }
4280   return default_assemble_integer (x, size, aligned_p);
4281 }
4282
4283 /* Emit the function prologue.  */
4284
4285 static void
4286 ia64_output_function_prologue (FILE *file, HOST_WIDE_INT size ATTRIBUTE_UNUSED)
4287 {
4288   int mask, grsave, grsave_prev;
4289
4290   if (current_frame_info.need_regstk)
4291     fprintf (file, "\t.regstk %d, %d, %d, %d\n",
4292              current_frame_info.n_input_regs,
4293              current_frame_info.n_local_regs,
4294              current_frame_info.n_output_regs,
4295              current_frame_info.n_rotate_regs);
4296
4297   if (ia64_except_unwind_info (&global_options) != UI_TARGET)
4298     return;
4299
4300   /* Emit the .prologue directive.  */
4301
4302   mask = 0;
4303   grsave = grsave_prev = 0;
4304   if (current_frame_info.r[reg_save_b0] != 0)
4305     {
4306       mask |= 8;
4307       grsave = grsave_prev = current_frame_info.r[reg_save_b0];
4308     }
4309   if (current_frame_info.r[reg_save_ar_pfs] != 0
4310       && (grsave_prev == 0
4311           || current_frame_info.r[reg_save_ar_pfs] == grsave_prev + 1))
4312     {
4313       mask |= 4;
4314       if (grsave_prev == 0)
4315         grsave = current_frame_info.r[reg_save_ar_pfs];
4316       grsave_prev = current_frame_info.r[reg_save_ar_pfs];
4317     }
4318   if (current_frame_info.r[reg_fp] != 0
4319       && (grsave_prev == 0
4320           || current_frame_info.r[reg_fp] == grsave_prev + 1))
4321     {
4322       mask |= 2;
4323       if (grsave_prev == 0)
4324         grsave = HARD_FRAME_POINTER_REGNUM;
4325       grsave_prev = current_frame_info.r[reg_fp];
4326     }
4327   if (current_frame_info.r[reg_save_pr] != 0
4328       && (grsave_prev == 0
4329           || current_frame_info.r[reg_save_pr] == grsave_prev + 1))
4330     {
4331       mask |= 1;
4332       if (grsave_prev == 0)
4333         grsave = current_frame_info.r[reg_save_pr];
4334     }
4335
4336   if (mask && TARGET_GNU_AS)
4337     fprintf (file, "\t.prologue %d, %d\n", mask,
4338              ia64_dbx_register_number (grsave));
4339   else
4340     fputs ("\t.prologue\n", file);
4341
4342   /* Emit a .spill directive, if necessary, to relocate the base of
4343      the register spill area.  */
4344   if (current_frame_info.spill_cfa_off != -16)
4345     fprintf (file, "\t.spill %ld\n",
4346              (long) (current_frame_info.spill_cfa_off
4347                      + current_frame_info.spill_size));
4348 }
4349
4350 /* Emit the .body directive at the scheduled end of the prologue.  */
4351
4352 static void
4353 ia64_output_function_end_prologue (FILE *file)
4354 {
4355   if (ia64_except_unwind_info (&global_options) != UI_TARGET)
4356     return;
4357
4358   fputs ("\t.body\n", file);
4359 }
4360
4361 /* Emit the function epilogue.  */
4362
4363 static void
4364 ia64_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED,
4365                                HOST_WIDE_INT size ATTRIBUTE_UNUSED)
4366 {
4367   int i;
4368
4369   if (current_frame_info.r[reg_fp])
4370     {
4371       const char *tmp = reg_names[HARD_FRAME_POINTER_REGNUM];
4372       reg_names[HARD_FRAME_POINTER_REGNUM]
4373         = reg_names[current_frame_info.r[reg_fp]];
4374       reg_names[current_frame_info.r[reg_fp]] = tmp;
4375       reg_emitted (reg_fp);
4376     }
4377   if (! TARGET_REG_NAMES)
4378     {
4379       for (i = 0; i < current_frame_info.n_input_regs; i++)
4380         reg_names[IN_REG (i)] = ia64_input_reg_names[i];
4381       for (i = 0; i < current_frame_info.n_local_regs; i++)
4382         reg_names[LOC_REG (i)] = ia64_local_reg_names[i];
4383       for (i = 0; i < current_frame_info.n_output_regs; i++)
4384         reg_names[OUT_REG (i)] = ia64_output_reg_names[i];
4385     }
4386
4387   current_frame_info.initialized = 0;
4388 }
4389
4390 int
4391 ia64_dbx_register_number (int regno)
4392 {
4393   /* In ia64_expand_prologue we quite literally renamed the frame pointer
4394      from its home at loc79 to something inside the register frame.  We
4395      must perform the same renumbering here for the debug info.  */
4396   if (current_frame_info.r[reg_fp])
4397     {
4398       if (regno == HARD_FRAME_POINTER_REGNUM)
4399         regno = current_frame_info.r[reg_fp];
4400       else if (regno == current_frame_info.r[reg_fp])
4401         regno = HARD_FRAME_POINTER_REGNUM;
4402     }
4403
4404   if (IN_REGNO_P (regno))
4405     return 32 + regno - IN_REG (0);
4406   else if (LOC_REGNO_P (regno))
4407     return 32 + current_frame_info.n_input_regs + regno - LOC_REG (0);
4408   else if (OUT_REGNO_P (regno))
4409     return (32 + current_frame_info.n_input_regs
4410             + current_frame_info.n_local_regs + regno - OUT_REG (0));
4411   else
4412     return regno;
4413 }
4414
4415 /* Implement TARGET_TRAMPOLINE_INIT.
4416
4417    The trampoline should set the static chain pointer to value placed
4418    into the trampoline and should branch to the specified routine.
4419    To make the normal indirect-subroutine calling convention work,
4420    the trampoline must look like a function descriptor; the first
4421    word being the target address and the second being the target's
4422    global pointer.
4423
4424    We abuse the concept of a global pointer by arranging for it
4425    to point to the data we need to load.  The complete trampoline
4426    has the following form:
4427
4428                 +-------------------+ \
4429         TRAMP:  | __ia64_trampoline | |
4430                 +-------------------+  > fake function descriptor
4431                 | TRAMP+16          | |
4432                 +-------------------+ /
4433                 | target descriptor |
4434                 +-------------------+
4435                 | static link       |
4436                 +-------------------+
4437 */
4438
4439 static void
4440 ia64_trampoline_init (rtx m_tramp, tree fndecl, rtx static_chain)
4441 {
4442   rtx fnaddr = XEXP (DECL_RTL (fndecl), 0);
4443   rtx addr, addr_reg, tramp, eight = GEN_INT (8);
4444
4445   /* The Intel assembler requires that the global __ia64_trampoline symbol
4446      be declared explicitly */
4447   if (!TARGET_GNU_AS)
4448     {
4449       static bool declared_ia64_trampoline = false;
4450
4451       if (!declared_ia64_trampoline)
4452         {
4453           declared_ia64_trampoline = true;
4454           (*targetm.asm_out.globalize_label) (asm_out_file,
4455                                               "__ia64_trampoline");
4456         }
4457     }
4458
4459   /* Make sure addresses are Pmode even if we are in ILP32 mode. */
4460   addr = convert_memory_address (Pmode, XEXP (m_tramp, 0));
4461   fnaddr = convert_memory_address (Pmode, fnaddr);
4462   static_chain = convert_memory_address (Pmode, static_chain);
4463
4464   /* Load up our iterator.  */
4465   addr_reg = copy_to_reg (addr);
4466   m_tramp = adjust_automodify_address (m_tramp, Pmode, addr_reg, 0);
4467
4468   /* The first two words are the fake descriptor:
4469      __ia64_trampoline, ADDR+16.  */
4470   tramp = gen_rtx_SYMBOL_REF (Pmode, "__ia64_trampoline");
4471   if (TARGET_ABI_OPEN_VMS)
4472     {
4473       /* HP decided to break the ELF ABI on VMS (to deal with an ambiguity
4474          in the Macro-32 compiler) and changed the semantics of the LTOFF22
4475          relocation against function symbols to make it identical to the
4476          LTOFF_FPTR22 relocation.  Emit the latter directly to stay within
4477          strict ELF and dereference to get the bare code address.  */
4478       rtx reg = gen_reg_rtx (Pmode);
4479       SYMBOL_REF_FLAGS (tramp) |= SYMBOL_FLAG_FUNCTION;
4480       emit_move_insn (reg, tramp);
4481       emit_move_insn (reg, gen_rtx_MEM (Pmode, reg));
4482       tramp = reg;
4483    }
4484   emit_move_insn (m_tramp, tramp);
4485   emit_insn (gen_adddi3 (addr_reg, addr_reg, eight));
4486   m_tramp = adjust_automodify_address (m_tramp, VOIDmode, NULL, 8);
4487
4488   emit_move_insn (m_tramp, force_reg (Pmode, plus_constant (Pmode, addr, 16)));
4489   emit_insn (gen_adddi3 (addr_reg, addr_reg, eight));
4490   m_tramp = adjust_automodify_address (m_tramp, VOIDmode, NULL, 8);
4491
4492   /* The third word is the target descriptor.  */
4493   emit_move_insn (m_tramp, force_reg (Pmode, fnaddr));
4494   emit_insn (gen_adddi3 (addr_reg, addr_reg, eight));
4495   m_tramp = adjust_automodify_address (m_tramp, VOIDmode, NULL, 8);
4496
4497   /* The fourth word is the static chain.  */
4498   emit_move_insn (m_tramp, static_chain);
4499 }
4500 \f
4501 /* Do any needed setup for a variadic function.  CUM has not been updated
4502    for the last named argument which has type TYPE and mode MODE.
4503
4504    We generate the actual spill instructions during prologue generation.  */
4505
4506 static void
4507 ia64_setup_incoming_varargs (cumulative_args_t cum, machine_mode mode,
4508                              tree type, int * pretend_size,
4509                              int second_time ATTRIBUTE_UNUSED)
4510 {
4511   CUMULATIVE_ARGS next_cum = *get_cumulative_args (cum);
4512
4513   /* Skip the current argument.  */
4514   ia64_function_arg_advance (pack_cumulative_args (&next_cum), mode, type, 1);
4515
4516   if (next_cum.words < MAX_ARGUMENT_SLOTS)
4517     {
4518       int n = MAX_ARGUMENT_SLOTS - next_cum.words;
4519       *pretend_size = n * UNITS_PER_WORD;
4520       cfun->machine->n_varargs = n;
4521     }
4522 }
4523
4524 /* Check whether TYPE is a homogeneous floating point aggregate.  If
4525    it is, return the mode of the floating point type that appears
4526    in all leafs.  If it is not, return VOIDmode.
4527
4528    An aggregate is a homogeneous floating point aggregate is if all
4529    fields/elements in it have the same floating point type (e.g,
4530    SFmode).  128-bit quad-precision floats are excluded.
4531
4532    Variable sized aggregates should never arrive here, since we should
4533    have already decided to pass them by reference.  Top-level zero-sized
4534    aggregates are excluded because our parallels crash the middle-end.  */
4535
4536 static machine_mode
4537 hfa_element_mode (const_tree type, bool nested)
4538 {
4539   machine_mode element_mode = VOIDmode;
4540   machine_mode mode;
4541   enum tree_code code = TREE_CODE (type);
4542   int know_element_mode = 0;
4543   tree t;
4544
4545   if (!nested && (!TYPE_SIZE (type) || integer_zerop (TYPE_SIZE (type))))
4546     return VOIDmode;
4547
4548   switch (code)
4549     {
4550     case VOID_TYPE:     case INTEGER_TYPE:      case ENUMERAL_TYPE:
4551     case BOOLEAN_TYPE:  case POINTER_TYPE:
4552     case OFFSET_TYPE:   case REFERENCE_TYPE:    case METHOD_TYPE:
4553     case LANG_TYPE:             case FUNCTION_TYPE:
4554       return VOIDmode;
4555
4556       /* Fortran complex types are supposed to be HFAs, so we need to handle
4557          gcc's COMPLEX_TYPEs as HFAs.  We need to exclude the integral complex
4558          types though.  */
4559     case COMPLEX_TYPE:
4560       if (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_COMPLEX_FLOAT
4561           && TYPE_MODE (type) != TCmode)
4562         return GET_MODE_INNER (TYPE_MODE (type));
4563       else
4564         return VOIDmode;
4565
4566     case REAL_TYPE:
4567       /* We want to return VOIDmode for raw REAL_TYPEs, but the actual
4568          mode if this is contained within an aggregate.  */
4569       if (nested && TYPE_MODE (type) != TFmode)
4570         return TYPE_MODE (type);
4571       else
4572         return VOIDmode;
4573
4574     case ARRAY_TYPE:
4575       return hfa_element_mode (TREE_TYPE (type), 1);
4576
4577     case RECORD_TYPE:
4578     case UNION_TYPE:
4579     case QUAL_UNION_TYPE:
4580       for (t = TYPE_FIELDS (type); t; t = DECL_CHAIN (t))
4581         {
4582           if (TREE_CODE (t) != FIELD_DECL)
4583             continue;
4584
4585           mode = hfa_element_mode (TREE_TYPE (t), 1);
4586           if (know_element_mode)
4587             {
4588               if (mode != element_mode)
4589                 return VOIDmode;
4590             }
4591           else if (GET_MODE_CLASS (mode) != MODE_FLOAT)
4592             return VOIDmode;
4593           else
4594             {
4595               know_element_mode = 1;
4596               element_mode = mode;
4597             }
4598         }
4599       return element_mode;
4600
4601     default:
4602       /* If we reach here, we probably have some front-end specific type
4603          that the backend doesn't know about.  This can happen via the
4604          aggregate_value_p call in init_function_start.  All we can do is
4605          ignore unknown tree types.  */
4606       return VOIDmode;
4607     }
4608
4609   return VOIDmode;
4610 }
4611
4612 /* Return the number of words required to hold a quantity of TYPE and MODE
4613    when passed as an argument.  */
4614 static int
4615 ia64_function_arg_words (const_tree type, machine_mode mode)
4616 {
4617   int words;
4618
4619   if (mode == BLKmode)
4620     words = int_size_in_bytes (type);
4621   else
4622     words = GET_MODE_SIZE (mode);
4623
4624   return (words + UNITS_PER_WORD - 1) / UNITS_PER_WORD;  /* round up */
4625 }
4626
4627 /* Return the number of registers that should be skipped so the current
4628    argument (described by TYPE and WORDS) will be properly aligned.
4629
4630    Integer and float arguments larger than 8 bytes start at the next
4631    even boundary.  Aggregates larger than 8 bytes start at the next
4632    even boundary if the aggregate has 16 byte alignment.  Note that
4633    in the 32-bit ABI, TImode and TFmode have only 8-byte alignment
4634    but are still to be aligned in registers.
4635
4636    ??? The ABI does not specify how to handle aggregates with
4637    alignment from 9 to 15 bytes, or greater than 16.  We handle them
4638    all as if they had 16 byte alignment.  Such aggregates can occur
4639    only if gcc extensions are used.  */
4640 static int
4641 ia64_function_arg_offset (const CUMULATIVE_ARGS *cum,
4642                           const_tree type, int words)
4643 {
4644   /* No registers are skipped on VMS.  */
4645   if (TARGET_ABI_OPEN_VMS || (cum->words & 1) == 0)
4646     return 0;
4647
4648   if (type
4649       && TREE_CODE (type) != INTEGER_TYPE
4650       && TREE_CODE (type) != REAL_TYPE)
4651     return TYPE_ALIGN (type) > 8 * BITS_PER_UNIT;
4652   else
4653     return words > 1;
4654 }
4655
4656 /* Return rtx for register where argument is passed, or zero if it is passed
4657    on the stack.  */
4658 /* ??? 128-bit quad-precision floats are always passed in general
4659    registers.  */
4660
4661 static rtx
4662 ia64_function_arg_1 (cumulative_args_t cum_v, machine_mode mode,
4663                      const_tree type, bool named, bool incoming)
4664 {
4665   const CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
4666
4667   int basereg = (incoming ? GR_ARG_FIRST : AR_ARG_FIRST);
4668   int words = ia64_function_arg_words (type, mode);
4669   int offset = ia64_function_arg_offset (cum, type, words);
4670   machine_mode hfa_mode = VOIDmode;
4671
4672   /* For OPEN VMS, emit the instruction setting up the argument register here,
4673      when we know this will be together with the other arguments setup related
4674      insns.  This is not the conceptually best place to do this, but this is
4675      the easiest as we have convenient access to cumulative args info.  */
4676
4677   if (TARGET_ABI_OPEN_VMS && mode == VOIDmode && type == void_type_node
4678       && named == 1)
4679     {
4680       unsigned HOST_WIDE_INT regval = cum->words;
4681       int i;
4682
4683       for (i = 0; i < 8; i++)
4684         regval |= ((int) cum->atypes[i]) << (i * 3 + 8);
4685
4686       emit_move_insn (gen_rtx_REG (DImode, GR_REG (25)),
4687                       GEN_INT (regval));
4688     }
4689
4690   /* If all argument slots are used, then it must go on the stack.  */
4691   if (cum->words + offset >= MAX_ARGUMENT_SLOTS)
4692     return 0;
4693
4694   /* On OpenVMS argument is either in Rn or Fn.  */
4695   if (TARGET_ABI_OPEN_VMS)
4696     {
4697       if (FLOAT_MODE_P (mode))
4698         return gen_rtx_REG (mode, FR_ARG_FIRST + cum->words);
4699       else
4700         return gen_rtx_REG (mode, basereg + cum->words);
4701     }
4702
4703   /* Check for and handle homogeneous FP aggregates.  */
4704   if (type)
4705     hfa_mode = hfa_element_mode (type, 0);
4706
4707   /* Unnamed prototyped hfas are passed as usual.  Named prototyped hfas
4708      and unprototyped hfas are passed specially.  */
4709   if (hfa_mode != VOIDmode && (! cum->prototype || named))
4710     {
4711       rtx loc[16];
4712       int i = 0;
4713       int fp_regs = cum->fp_regs;
4714       int int_regs = cum->words + offset;
4715       int hfa_size = GET_MODE_SIZE (hfa_mode);
4716       int byte_size;
4717       int args_byte_size;
4718
4719       /* If prototyped, pass it in FR regs then GR regs.
4720          If not prototyped, pass it in both FR and GR regs.
4721
4722          If this is an SFmode aggregate, then it is possible to run out of
4723          FR regs while GR regs are still left.  In that case, we pass the
4724          remaining part in the GR regs.  */
4725
4726       /* Fill the FP regs.  We do this always.  We stop if we reach the end
4727          of the argument, the last FP register, or the last argument slot.  */
4728
4729       byte_size = ((mode == BLKmode)
4730                    ? int_size_in_bytes (type) : GET_MODE_SIZE (mode));
4731       args_byte_size = int_regs * UNITS_PER_WORD;
4732       offset = 0;
4733       for (; (offset < byte_size && fp_regs < MAX_ARGUMENT_SLOTS
4734               && args_byte_size < (MAX_ARGUMENT_SLOTS * UNITS_PER_WORD)); i++)
4735         {
4736           loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
4737                                       gen_rtx_REG (hfa_mode, (FR_ARG_FIRST
4738                                                               + fp_regs)),
4739                                       GEN_INT (offset));
4740           offset += hfa_size;
4741           args_byte_size += hfa_size;
4742           fp_regs++;
4743         }
4744
4745       /* If no prototype, then the whole thing must go in GR regs.  */
4746       if (! cum->prototype)
4747         offset = 0;
4748       /* If this is an SFmode aggregate, then we might have some left over
4749          that needs to go in GR regs.  */
4750       else if (byte_size != offset)
4751         int_regs += offset / UNITS_PER_WORD;
4752
4753       /* Fill in the GR regs.  We must use DImode here, not the hfa mode.  */
4754
4755       for (; offset < byte_size && int_regs < MAX_ARGUMENT_SLOTS; i++)
4756         {
4757           machine_mode gr_mode = DImode;
4758           unsigned int gr_size;
4759
4760           /* If we have an odd 4 byte hunk because we ran out of FR regs,
4761              then this goes in a GR reg left adjusted/little endian, right
4762              adjusted/big endian.  */
4763           /* ??? Currently this is handled wrong, because 4-byte hunks are
4764              always right adjusted/little endian.  */
4765           if (offset & 0x4)
4766             gr_mode = SImode;
4767           /* If we have an even 4 byte hunk because the aggregate is a
4768              multiple of 4 bytes in size, then this goes in a GR reg right
4769              adjusted/little endian.  */
4770           else if (byte_size - offset == 4)
4771             gr_mode = SImode;
4772
4773           loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
4774                                       gen_rtx_REG (gr_mode, (basereg
4775                                                              + int_regs)),
4776                                       GEN_INT (offset));
4777
4778           gr_size = GET_MODE_SIZE (gr_mode);
4779           offset += gr_size;
4780           if (gr_size == UNITS_PER_WORD
4781               || (gr_size < UNITS_PER_WORD && offset % UNITS_PER_WORD == 0))
4782             int_regs++;
4783           else if (gr_size > UNITS_PER_WORD)
4784             int_regs += gr_size / UNITS_PER_WORD;
4785         }
4786       return gen_rtx_PARALLEL (mode, gen_rtvec_v (i, loc));
4787     }
4788
4789   /* Integral and aggregates go in general registers.  If we have run out of
4790      FR registers, then FP values must also go in general registers.  This can
4791      happen when we have a SFmode HFA.  */
4792   else if (mode == TFmode || mode == TCmode
4793            || (! FLOAT_MODE_P (mode) || cum->fp_regs == MAX_ARGUMENT_SLOTS))
4794     {
4795       int byte_size = ((mode == BLKmode)
4796                        ? int_size_in_bytes (type) : GET_MODE_SIZE (mode));
4797       if (BYTES_BIG_ENDIAN
4798         && (mode == BLKmode || (type && AGGREGATE_TYPE_P (type)))
4799         && byte_size < UNITS_PER_WORD
4800         && byte_size > 0)
4801         {
4802           rtx gr_reg = gen_rtx_EXPR_LIST (VOIDmode,
4803                                           gen_rtx_REG (DImode,
4804                                                        (basereg + cum->words
4805                                                         + offset)),
4806                                           const0_rtx);
4807           return gen_rtx_PARALLEL (mode, gen_rtvec (1, gr_reg));
4808         }
4809       else
4810         return gen_rtx_REG (mode, basereg + cum->words + offset);
4811
4812     }
4813
4814   /* If there is a prototype, then FP values go in a FR register when
4815      named, and in a GR register when unnamed.  */
4816   else if (cum->prototype)
4817     {
4818       if (named)
4819         return gen_rtx_REG (mode, FR_ARG_FIRST + cum->fp_regs);
4820       /* In big-endian mode, an anonymous SFmode value must be represented
4821          as (parallel:SF [(expr_list (reg:DI n) (const_int 0))]) to force
4822          the value into the high half of the general register.  */
4823       else if (BYTES_BIG_ENDIAN && mode == SFmode)
4824         return gen_rtx_PARALLEL (mode,
4825                  gen_rtvec (1,
4826                    gen_rtx_EXPR_LIST (VOIDmode,
4827                      gen_rtx_REG (DImode, basereg + cum->words + offset),
4828                                       const0_rtx)));
4829       else
4830         return gen_rtx_REG (mode, basereg + cum->words + offset);
4831     }
4832   /* If there is no prototype, then FP values go in both FR and GR
4833      registers.  */
4834   else
4835     {
4836       /* See comment above.  */
4837       machine_mode inner_mode =
4838         (BYTES_BIG_ENDIAN && mode == SFmode) ? DImode : mode;
4839
4840       rtx fp_reg = gen_rtx_EXPR_LIST (VOIDmode,
4841                                       gen_rtx_REG (mode, (FR_ARG_FIRST
4842                                                           + cum->fp_regs)),
4843                                       const0_rtx);
4844       rtx gr_reg = gen_rtx_EXPR_LIST (VOIDmode,
4845                                       gen_rtx_REG (inner_mode,
4846                                                    (basereg + cum->words
4847                                                     + offset)),
4848                                       const0_rtx);
4849
4850       return gen_rtx_PARALLEL (mode, gen_rtvec (2, fp_reg, gr_reg));
4851     }
4852 }
4853
4854 /* Implement TARGET_FUNCION_ARG target hook.  */
4855
4856 static rtx
4857 ia64_function_arg (cumulative_args_t cum, machine_mode mode,
4858                    const_tree type, bool named)
4859 {
4860   return ia64_function_arg_1 (cum, mode, type, named, false);
4861 }
4862
4863 /* Implement TARGET_FUNCION_INCOMING_ARG target hook.  */
4864
4865 static rtx
4866 ia64_function_incoming_arg (cumulative_args_t cum,
4867                             machine_mode mode,
4868                             const_tree type, bool named)
4869 {
4870   return ia64_function_arg_1 (cum, mode, type, named, true);
4871 }
4872
4873 /* Return number of bytes, at the beginning of the argument, that must be
4874    put in registers.  0 is the argument is entirely in registers or entirely
4875    in memory.  */
4876
4877 static int
4878 ia64_arg_partial_bytes (cumulative_args_t cum_v, machine_mode mode,
4879                         tree type, bool named ATTRIBUTE_UNUSED)
4880 {
4881   CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
4882
4883   int words = ia64_function_arg_words (type, mode);
4884   int offset = ia64_function_arg_offset (cum, type, words);
4885
4886   /* If all argument slots are used, then it must go on the stack.  */
4887   if (cum->words + offset >= MAX_ARGUMENT_SLOTS)
4888     return 0;
4889
4890   /* It doesn't matter whether the argument goes in FR or GR regs.  If
4891      it fits within the 8 argument slots, then it goes entirely in
4892      registers.  If it extends past the last argument slot, then the rest
4893      goes on the stack.  */
4894
4895   if (words + cum->words + offset <= MAX_ARGUMENT_SLOTS)
4896     return 0;
4897
4898   return (MAX_ARGUMENT_SLOTS - cum->words - offset) * UNITS_PER_WORD;
4899 }
4900
4901 /* Return ivms_arg_type based on machine_mode.  */
4902
4903 static enum ivms_arg_type
4904 ia64_arg_type (machine_mode mode)
4905 {
4906   switch (mode)
4907     {
4908     case SFmode:
4909       return FS;
4910     case DFmode:
4911       return FT;
4912     default:
4913       return I64;
4914     }
4915 }
4916
4917 /* Update CUM to point after this argument.  This is patterned after
4918    ia64_function_arg.  */
4919
4920 static void
4921 ia64_function_arg_advance (cumulative_args_t cum_v, machine_mode mode,
4922                            const_tree type, bool named)
4923 {
4924   CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v);
4925   int words = ia64_function_arg_words (type, mode);
4926   int offset = ia64_function_arg_offset (cum, type, words);
4927   machine_mode hfa_mode = VOIDmode;
4928
4929   /* If all arg slots are already full, then there is nothing to do.  */
4930   if (cum->words >= MAX_ARGUMENT_SLOTS)
4931     {
4932       cum->words += words + offset;
4933       return;
4934     }
4935
4936   cum->atypes[cum->words] = ia64_arg_type (mode);
4937   cum->words += words + offset;
4938
4939   /* On OpenVMS argument is either in Rn or Fn.  */
4940   if (TARGET_ABI_OPEN_VMS)
4941     {
4942       cum->int_regs = cum->words;
4943       cum->fp_regs = cum->words;
4944       return;
4945     }
4946
4947   /* Check for and handle homogeneous FP aggregates.  */
4948   if (type)
4949     hfa_mode = hfa_element_mode (type, 0);
4950
4951   /* Unnamed prototyped hfas are passed as usual.  Named prototyped hfas
4952      and unprototyped hfas are passed specially.  */
4953   if (hfa_mode != VOIDmode && (! cum->prototype || named))
4954     {
4955       int fp_regs = cum->fp_regs;
4956       /* This is the original value of cum->words + offset.  */
4957       int int_regs = cum->words - words;
4958       int hfa_size = GET_MODE_SIZE (hfa_mode);
4959       int byte_size;
4960       int args_byte_size;
4961
4962       /* If prototyped, pass it in FR regs then GR regs.
4963          If not prototyped, pass it in both FR and GR regs.
4964
4965          If this is an SFmode aggregate, then it is possible to run out of
4966          FR regs while GR regs are still left.  In that case, we pass the
4967          remaining part in the GR regs.  */
4968
4969       /* Fill the FP regs.  We do this always.  We stop if we reach the end
4970          of the argument, the last FP register, or the last argument slot.  */
4971
4972       byte_size = ((mode == BLKmode)
4973                    ? int_size_in_bytes (type) : GET_MODE_SIZE (mode));
4974       args_byte_size = int_regs * UNITS_PER_WORD;
4975       offset = 0;
4976       for (; (offset < byte_size && fp_regs < MAX_ARGUMENT_SLOTS
4977               && args_byte_size < (MAX_ARGUMENT_SLOTS * UNITS_PER_WORD));)
4978         {
4979           offset += hfa_size;
4980           args_byte_size += hfa_size;
4981           fp_regs++;
4982         }
4983
4984       cum->fp_regs = fp_regs;
4985     }
4986
4987   /* Integral and aggregates go in general registers.  So do TFmode FP values.
4988      If we have run out of FR registers, then other FP values must also go in
4989      general registers.  This can happen when we have a SFmode HFA.  */
4990   else if (mode == TFmode || mode == TCmode
4991            || (! FLOAT_MODE_P (mode) || cum->fp_regs == MAX_ARGUMENT_SLOTS))
4992     cum->int_regs = cum->words;
4993
4994   /* If there is a prototype, then FP values go in a FR register when
4995      named, and in a GR register when unnamed.  */
4996   else if (cum->prototype)
4997     {
4998       if (! named)
4999         cum->int_regs = cum->words;
5000       else
5001         /* ??? Complex types should not reach here.  */
5002         cum->fp_regs += (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT ? 2 : 1);
5003     }
5004   /* If there is no prototype, then FP values go in both FR and GR
5005      registers.  */
5006   else
5007     {
5008       /* ??? Complex types should not reach here.  */
5009       cum->fp_regs += (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT ? 2 : 1);
5010       cum->int_regs = cum->words;
5011     }
5012 }
5013
5014 /* Arguments with alignment larger than 8 bytes start at the next even
5015    boundary.  On ILP32 HPUX, TFmode arguments start on next even boundary
5016    even though their normal alignment is 8 bytes.  See ia64_function_arg.  */
5017
5018 static unsigned int
5019 ia64_function_arg_boundary (machine_mode mode, const_tree type)
5020 {
5021   if (mode == TFmode && TARGET_HPUX && TARGET_ILP32)
5022     return PARM_BOUNDARY * 2;
5023
5024   if (type)
5025     {
5026       if (TYPE_ALIGN (type) > PARM_BOUNDARY)
5027         return PARM_BOUNDARY * 2;
5028       else
5029         return PARM_BOUNDARY;
5030     }
5031
5032   if (GET_MODE_BITSIZE (mode) > PARM_BOUNDARY)
5033     return PARM_BOUNDARY * 2;
5034   else
5035     return PARM_BOUNDARY;
5036 }
5037
5038 /* True if it is OK to do sibling call optimization for the specified
5039    call expression EXP.  DECL will be the called function, or NULL if
5040    this is an indirect call.  */
5041 static bool
5042 ia64_function_ok_for_sibcall (tree decl, tree exp ATTRIBUTE_UNUSED)
5043 {
5044   /* We can't perform a sibcall if the current function has the syscall_linkage
5045      attribute.  */
5046   if (lookup_attribute ("syscall_linkage",
5047                         TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
5048     return false;
5049
5050   /* We must always return with our current GP.  This means we can
5051      only sibcall to functions defined in the current module unless
5052      TARGET_CONST_GP is set to true.  */
5053   return (decl && (*targetm.binds_local_p) (decl)) || TARGET_CONST_GP;
5054 }
5055 \f
5056
5057 /* Implement va_arg.  */
5058
5059 static tree
5060 ia64_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
5061                       gimple_seq *post_p)
5062 {
5063   /* Variable sized types are passed by reference.  */
5064   if (pass_by_reference (NULL, TYPE_MODE (type), type, false))
5065     {
5066       tree ptrtype = build_pointer_type (type);
5067       tree addr = std_gimplify_va_arg_expr (valist, ptrtype, pre_p, post_p);
5068       return build_va_arg_indirect_ref (addr);
5069     }
5070
5071   /* Aggregate arguments with alignment larger than 8 bytes start at
5072      the next even boundary.  Integer and floating point arguments
5073      do so if they are larger than 8 bytes, whether or not they are
5074      also aligned larger than 8 bytes.  */
5075   if ((TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == INTEGER_TYPE)
5076       ? int_size_in_bytes (type) > 8 : TYPE_ALIGN (type) > 8 * BITS_PER_UNIT)
5077     {
5078       tree t = fold_build_pointer_plus_hwi (valist, 2 * UNITS_PER_WORD - 1);
5079       t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
5080                   build_int_cst (TREE_TYPE (t), -2 * UNITS_PER_WORD));
5081       gimplify_assign (unshare_expr (valist), t, pre_p);
5082     }
5083
5084   return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
5085 }
5086 \f
5087 /* Return 1 if function return value returned in memory.  Return 0 if it is
5088    in a register.  */
5089
5090 static bool
5091 ia64_return_in_memory (const_tree valtype, const_tree fntype ATTRIBUTE_UNUSED)
5092 {
5093   machine_mode mode;
5094   machine_mode hfa_mode;
5095   HOST_WIDE_INT byte_size;
5096
5097   mode = TYPE_MODE (valtype);
5098   byte_size = GET_MODE_SIZE (mode);
5099   if (mode == BLKmode)
5100     {
5101       byte_size = int_size_in_bytes (valtype);
5102       if (byte_size < 0)
5103         return true;
5104     }
5105
5106   /* Hfa's with up to 8 elements are returned in the FP argument registers.  */
5107
5108   hfa_mode = hfa_element_mode (valtype, 0);
5109   if (hfa_mode != VOIDmode)
5110     {
5111       int hfa_size = GET_MODE_SIZE (hfa_mode);
5112
5113       if (byte_size / hfa_size > MAX_ARGUMENT_SLOTS)
5114         return true;
5115       else
5116         return false;
5117     }
5118   else if (byte_size > UNITS_PER_WORD * MAX_INT_RETURN_SLOTS)
5119     return true;
5120   else
5121     return false;
5122 }
5123
5124 /* Return rtx for register that holds the function return value.  */
5125
5126 static rtx
5127 ia64_function_value (const_tree valtype,
5128                      const_tree fn_decl_or_type,
5129                      bool outgoing ATTRIBUTE_UNUSED)
5130 {
5131   machine_mode mode;
5132   machine_mode hfa_mode;
5133   int unsignedp;
5134   const_tree func = fn_decl_or_type;
5135
5136   if (fn_decl_or_type
5137       && !DECL_P (fn_decl_or_type))
5138     func = NULL;
5139
5140   mode = TYPE_MODE (valtype);
5141   hfa_mode = hfa_element_mode (valtype, 0);
5142
5143   if (hfa_mode != VOIDmode)
5144     {
5145       rtx loc[8];
5146       int i;
5147       int hfa_size;
5148       int byte_size;
5149       int offset;
5150
5151       hfa_size = GET_MODE_SIZE (hfa_mode);
5152       byte_size = ((mode == BLKmode)
5153                    ? int_size_in_bytes (valtype) : GET_MODE_SIZE (mode));
5154       offset = 0;
5155       for (i = 0; offset < byte_size; i++)
5156         {
5157           loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
5158                                       gen_rtx_REG (hfa_mode, FR_ARG_FIRST + i),
5159                                       GEN_INT (offset));
5160           offset += hfa_size;
5161         }
5162       return gen_rtx_PARALLEL (mode, gen_rtvec_v (i, loc));
5163     }
5164   else if (FLOAT_TYPE_P (valtype) && mode != TFmode && mode != TCmode)
5165     return gen_rtx_REG (mode, FR_ARG_FIRST);
5166   else
5167     {
5168       bool need_parallel = false;
5169
5170       /* In big-endian mode, we need to manage the layout of aggregates
5171          in the registers so that we get the bits properly aligned in
5172          the highpart of the registers.  */
5173       if (BYTES_BIG_ENDIAN
5174           && (mode == BLKmode || (valtype && AGGREGATE_TYPE_P (valtype))))
5175         need_parallel = true;
5176
5177       /* Something like struct S { long double x; char a[0] } is not an
5178          HFA structure, and therefore doesn't go in fp registers.  But
5179          the middle-end will give it XFmode anyway, and XFmode values
5180          don't normally fit in integer registers.  So we need to smuggle
5181          the value inside a parallel.  */
5182       else if (mode == XFmode || mode == XCmode || mode == RFmode)
5183         need_parallel = true;
5184
5185       if (need_parallel)
5186         {
5187           rtx loc[8];
5188           int offset;
5189           int bytesize;
5190           int i;
5191
5192           offset = 0;
5193           bytesize = int_size_in_bytes (valtype);
5194           /* An empty PARALLEL is invalid here, but the return value
5195              doesn't matter for empty structs.  */
5196           if (bytesize == 0)
5197             return gen_rtx_REG (mode, GR_RET_FIRST);
5198           for (i = 0; offset < bytesize; i++)
5199             {
5200               loc[i] = gen_rtx_EXPR_LIST (VOIDmode,
5201                                           gen_rtx_REG (DImode,
5202                                                        GR_RET_FIRST + i),
5203                                           GEN_INT (offset));
5204               offset += UNITS_PER_WORD;
5205             }
5206           return gen_rtx_PARALLEL (mode, gen_rtvec_v (i, loc));
5207         }
5208
5209       mode = promote_function_mode (valtype, mode, &unsignedp,
5210                                     func ? TREE_TYPE (func) : NULL_TREE,
5211                                     true);
5212
5213       return gen_rtx_REG (mode, GR_RET_FIRST);
5214     }
5215 }
5216
5217 /* Worker function for TARGET_LIBCALL_VALUE.  */
5218
5219 static rtx
5220 ia64_libcall_value (machine_mode mode,
5221                     const_rtx fun ATTRIBUTE_UNUSED)
5222 {
5223   return gen_rtx_REG (mode,
5224                       (((GET_MODE_CLASS (mode) == MODE_FLOAT
5225                          || GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT)
5226                         && (mode) != TFmode)
5227                        ? FR_RET_FIRST : GR_RET_FIRST));
5228 }
5229
5230 /* Worker function for FUNCTION_VALUE_REGNO_P.  */
5231
5232 static bool
5233 ia64_function_value_regno_p (const unsigned int regno)
5234 {
5235   return ((regno >= GR_RET_FIRST && regno <= GR_RET_LAST)
5236           || (regno >= FR_RET_FIRST && regno <= FR_RET_LAST));
5237 }
5238
5239 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
5240    We need to emit DTP-relative relocations.  */
5241
5242 static void
5243 ia64_output_dwarf_dtprel (FILE *file, int size, rtx x)
5244 {
5245   gcc_assert (size == 4 || size == 8);
5246   if (size == 4)
5247     fputs ("\tdata4.ua\t@dtprel(", file);
5248   else
5249     fputs ("\tdata8.ua\t@dtprel(", file);
5250   output_addr_const (file, x);
5251   fputs (")", file);
5252 }
5253
5254 /* Print a memory address as an operand to reference that memory location.  */
5255
5256 /* ??? Do we need this?  It gets used only for 'a' operands.  We could perhaps
5257    also call this from ia64_print_operand for memory addresses.  */
5258
5259 static void
5260 ia64_print_operand_address (FILE * stream ATTRIBUTE_UNUSED,
5261                             rtx address ATTRIBUTE_UNUSED)
5262 {
5263 }
5264
5265 /* Print an operand to an assembler instruction.
5266    C    Swap and print a comparison operator.
5267    D    Print an FP comparison operator.
5268    E    Print 32 - constant, for SImode shifts as extract.
5269    e    Print 64 - constant, for DImode rotates.
5270    F    A floating point constant 0.0 emitted as f0, or 1.0 emitted as f1, or
5271         a floating point register emitted normally.
5272    G    A floating point constant.
5273    I    Invert a predicate register by adding 1.
5274    J    Select the proper predicate register for a condition.
5275    j    Select the inverse predicate register for a condition.
5276    O    Append .acq for volatile load.
5277    P    Postincrement of a MEM.
5278    Q    Append .rel for volatile store.
5279    R    Print .s .d or nothing for a single, double or no truncation.
5280    S    Shift amount for shladd instruction.
5281    T    Print an 8-bit sign extended number (K) as a 32-bit unsigned number
5282         for Intel assembler.
5283    U    Print an 8-bit sign extended number (K) as a 64-bit unsigned number
5284         for Intel assembler.
5285    X    A pair of floating point registers.
5286    r    Print register name, or constant 0 as r0.  HP compatibility for
5287         Linux kernel.
5288    v    Print vector constant value as an 8-byte integer value.  */
5289
5290 static void
5291 ia64_print_operand (FILE * file, rtx x, int code)
5292 {
5293   const char *str;
5294
5295   switch (code)
5296     {
5297     case 0:
5298       /* Handled below.  */
5299       break;
5300
5301     case 'C':
5302       {
5303         enum rtx_code c = swap_condition (GET_CODE (x));
5304         fputs (GET_RTX_NAME (c), file);
5305         return;
5306       }
5307
5308     case 'D':
5309       switch (GET_CODE (x))
5310         {
5311         case NE:
5312           str = "neq";
5313           break;
5314         case UNORDERED:
5315           str = "unord";
5316           break;
5317         case ORDERED:
5318           str = "ord";
5319           break;
5320         case UNLT:
5321           str = "nge";
5322           break;
5323         case UNLE:
5324           str = "ngt";
5325           break;
5326         case UNGT:
5327           str = "nle";
5328           break;
5329         case UNGE:
5330           str = "nlt";
5331           break;
5332         case UNEQ:
5333         case LTGT:
5334           gcc_unreachable ();
5335         default:
5336           str = GET_RTX_NAME (GET_CODE (x));
5337           break;
5338         }
5339       fputs (str, file);
5340       return;
5341
5342     case 'E':
5343       fprintf (file, HOST_WIDE_INT_PRINT_DEC, 32 - INTVAL (x));
5344       return;
5345
5346     case 'e':
5347       fprintf (file, HOST_WIDE_INT_PRINT_DEC, 64 - INTVAL (x));
5348       return;
5349
5350     case 'F':
5351       if (x == CONST0_RTX (GET_MODE (x)))
5352         str = reg_names [FR_REG (0)];
5353       else if (x == CONST1_RTX (GET_MODE (x)))
5354         str = reg_names [FR_REG (1)];
5355       else
5356         {
5357           gcc_assert (GET_CODE (x) == REG);
5358           str = reg_names [REGNO (x)];
5359         }
5360       fputs (str, file);
5361       return;
5362
5363     case 'G':
5364       {
5365         long val[4];
5366         REAL_VALUE_TYPE rv;
5367         REAL_VALUE_FROM_CONST_DOUBLE (rv, x);
5368         real_to_target (val, &rv, GET_MODE (x));
5369         if (GET_MODE (x) == SFmode)
5370           fprintf (file, "0x%08lx", val[0] & 0xffffffff);
5371         else if (GET_MODE (x) == DFmode)
5372           fprintf (file, "0x%08lx%08lx", (WORDS_BIG_ENDIAN ? val[0] : val[1])
5373                                           & 0xffffffff,
5374                                          (WORDS_BIG_ENDIAN ? val[1] : val[0])
5375                                           & 0xffffffff);
5376         else
5377           output_operand_lossage ("invalid %%G mode");
5378       }
5379       return;
5380
5381     case 'I':
5382       fputs (reg_names [REGNO (x) + 1], file);
5383       return;
5384
5385     case 'J':
5386     case 'j':
5387       {
5388         unsigned int regno = REGNO (XEXP (x, 0));
5389         if (GET_CODE (x) == EQ)
5390           regno += 1;
5391         if (code == 'j')
5392           regno ^= 1;
5393         fputs (reg_names [regno], file);
5394       }
5395       return;
5396
5397     case 'O':
5398       if (MEM_VOLATILE_P (x))
5399         fputs(".acq", file);
5400       return;
5401
5402     case 'P':
5403       {
5404         HOST_WIDE_INT value;
5405
5406         switch (GET_CODE (XEXP (x, 0)))
5407           {
5408           default:
5409             return;
5410
5411           case POST_MODIFY:
5412             x = XEXP (XEXP (XEXP (x, 0), 1), 1);
5413             if (GET_CODE (x) == CONST_INT)
5414               value = INTVAL (x);
5415             else
5416               {
5417                 gcc_assert (GET_CODE (x) == REG);
5418                 fprintf (file, ", %s", reg_names[REGNO (x)]);
5419                 return;
5420               }
5421             break;
5422
5423           case POST_INC:
5424             value = GET_MODE_SIZE (GET_MODE (x));
5425             break;
5426
5427           case POST_DEC:
5428             value = - (HOST_WIDE_INT) GET_MODE_SIZE (GET_MODE (x));
5429             break;
5430           }
5431
5432         fprintf (file, ", " HOST_WIDE_INT_PRINT_DEC, value);
5433         return;
5434       }
5435
5436     case 'Q':
5437       if (MEM_VOLATILE_P (x))
5438         fputs(".rel", file);
5439       return;
5440
5441     case 'R':
5442       if (x == CONST0_RTX (GET_MODE (x)))
5443         fputs(".s", file);
5444       else if (x == CONST1_RTX (GET_MODE (x)))
5445         fputs(".d", file);
5446       else if (x == CONST2_RTX (GET_MODE (x)))
5447         ;
5448       else
5449         output_operand_lossage ("invalid %%R value");
5450       return;
5451
5452     case 'S':
5453       fprintf (file, "%d", exact_log2 (INTVAL (x)));
5454       return;
5455
5456     case 'T':
5457       if (! TARGET_GNU_AS && GET_CODE (x) == CONST_INT)
5458         {
5459           fprintf (file, "0x%x", (int) INTVAL (x) & 0xffffffff);
5460           return;
5461         }
5462       break;
5463
5464     case 'U':
5465       if (! TARGET_GNU_AS && GET_CODE (x) == CONST_INT)
5466         {
5467           const char *prefix = "0x";
5468           if (INTVAL (x) & 0x80000000)
5469             {
5470               fprintf (file, "0xffffffff");
5471               prefix = "";
5472             }
5473           fprintf (file, "%s%x", prefix, (int) INTVAL (x) & 0xffffffff);
5474           return;
5475         }
5476       break;
5477
5478     case 'X':
5479       {
5480         unsigned int regno = REGNO (x);
5481         fprintf (file, "%s, %s", reg_names [regno], reg_names [regno + 1]);
5482       }
5483       return;
5484
5485     case 'r':
5486       /* If this operand is the constant zero, write it as register zero.
5487          Any register, zero, or CONST_INT value is OK here.  */
5488       if (GET_CODE (x) == REG)
5489         fputs (reg_names[REGNO (x)], file);
5490       else if (x == CONST0_RTX (GET_MODE (x)))
5491         fputs ("r0", file);
5492       else if (GET_CODE (x) == CONST_INT)
5493         output_addr_const (file, x);
5494       else
5495         output_operand_lossage ("invalid %%r value");
5496       return;
5497
5498     case 'v':
5499       gcc_assert (GET_CODE (x) == CONST_VECTOR);
5500       x = simplify_subreg (DImode, x, GET_MODE (x), 0);
5501       break;
5502
5503     case '+':
5504       {
5505         const char *which;
5506
5507         /* For conditional branches, returns or calls, substitute
5508            sptk, dptk, dpnt, or spnt for %s.  */
5509         x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
5510         if (x)
5511           {
5512             int pred_val = XINT (x, 0);
5513
5514             /* Guess top and bottom 10% statically predicted.  */
5515             if (pred_val < REG_BR_PROB_BASE / 50
5516                 && br_prob_note_reliable_p (x))
5517               which = ".spnt";
5518             else if (pred_val < REG_BR_PROB_BASE / 2)
5519               which = ".dpnt";
5520             else if (pred_val < REG_BR_PROB_BASE / 100 * 98
5521                      || !br_prob_note_reliable_p (x))
5522               which = ".dptk";
5523             else
5524               which = ".sptk";
5525           }
5526         else if (CALL_P (current_output_insn))
5527           which = ".sptk";
5528         else
5529           which = ".dptk";
5530
5531         fputs (which, file);
5532         return;
5533       }
5534
5535     case ',':
5536       x = current_insn_predicate;
5537       if (x)
5538         {
5539           unsigned int regno = REGNO (XEXP (x, 0));
5540           if (GET_CODE (x) == EQ)
5541             regno += 1;
5542           fprintf (file, "(%s) ", reg_names [regno]);
5543         }
5544       return;
5545
5546     default:
5547       output_operand_lossage ("ia64_print_operand: unknown code");
5548       return;
5549     }
5550
5551   switch (GET_CODE (x))
5552     {
5553       /* This happens for the spill/restore instructions.  */
5554     case POST_INC:
5555     case POST_DEC:
5556     case POST_MODIFY:
5557       x = XEXP (x, 0);
5558       /* ... fall through ...  */
5559
5560     case REG:
5561       fputs (reg_names [REGNO (x)], file);
5562       break;
5563
5564     case MEM:
5565       {
5566         rtx addr = XEXP (x, 0);
5567         if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
5568           addr = XEXP (addr, 0);
5569         fprintf (file, "[%s]", reg_names [REGNO (addr)]);
5570         break;
5571       }
5572
5573     default:
5574       output_addr_const (file, x);
5575       break;
5576     }
5577
5578   return;
5579 }
5580
5581 /* Worker function for TARGET_PRINT_OPERAND_PUNCT_VALID_P.  */
5582
5583 static bool
5584 ia64_print_operand_punct_valid_p (unsigned char code)
5585 {
5586   return (code == '+' || code == ',');
5587 }
5588 \f
5589 /* Compute a (partial) cost for rtx X.  Return true if the complete
5590    cost has been computed, and false if subexpressions should be
5591    scanned.  In either case, *TOTAL contains the cost result.  */
5592 /* ??? This is incomplete.  */
5593
5594 static bool
5595 ia64_rtx_costs (rtx x, int code, int outer_code, int opno ATTRIBUTE_UNUSED,
5596                 int *total, bool speed ATTRIBUTE_UNUSED)
5597 {
5598   switch (code)
5599     {
5600     case CONST_INT:
5601       switch (outer_code)
5602         {
5603         case SET:
5604           *total = satisfies_constraint_J (x) ? 0 : COSTS_N_INSNS (1);
5605           return true;
5606         case PLUS:
5607           if (satisfies_constraint_I (x))
5608             *total = 0;
5609           else if (satisfies_constraint_J (x))
5610             *total = 1;
5611           else
5612             *total = COSTS_N_INSNS (1);
5613           return true;
5614         default:
5615           if (satisfies_constraint_K (x) || satisfies_constraint_L (x))
5616             *total = 0;
5617           else
5618             *total = COSTS_N_INSNS (1);
5619           return true;
5620         }
5621
5622     case CONST_DOUBLE:
5623       *total = COSTS_N_INSNS (1);
5624       return true;
5625
5626     case CONST:
5627     case SYMBOL_REF:
5628     case LABEL_REF:
5629       *total = COSTS_N_INSNS (3);
5630       return true;
5631
5632     case FMA:
5633       *total = COSTS_N_INSNS (4);
5634       return true;
5635
5636     case MULT:
5637       /* For multiplies wider than HImode, we have to go to the FPU,
5638          which normally involves copies.  Plus there's the latency
5639          of the multiply itself, and the latency of the instructions to
5640          transfer integer regs to FP regs.  */
5641       if (FLOAT_MODE_P (GET_MODE (x)))
5642         *total = COSTS_N_INSNS (4);
5643       else if (GET_MODE_SIZE (GET_MODE (x)) > 2)
5644         *total = COSTS_N_INSNS (10);
5645       else
5646         *total = COSTS_N_INSNS (2);
5647       return true;
5648
5649     case PLUS:
5650     case MINUS:
5651       if (FLOAT_MODE_P (GET_MODE (x)))
5652         {
5653           *total = COSTS_N_INSNS (4);
5654           return true;
5655         }
5656       /* FALLTHRU */
5657
5658     case ASHIFT:
5659     case ASHIFTRT:
5660     case LSHIFTRT:
5661       *total = COSTS_N_INSNS (1);
5662       return true;
5663
5664     case DIV:
5665     case UDIV:
5666     case MOD:
5667     case UMOD:
5668       /* We make divide expensive, so that divide-by-constant will be
5669          optimized to a multiply.  */
5670       *total = COSTS_N_INSNS (60);
5671       return true;
5672
5673     default:
5674       return false;
5675     }
5676 }
5677
5678 /* Calculate the cost of moving data from a register in class FROM to
5679    one in class TO, using MODE.  */
5680
5681 static int
5682 ia64_register_move_cost (machine_mode mode, reg_class_t from,
5683                          reg_class_t to)
5684 {
5685   /* ADDL_REGS is the same as GR_REGS for movement purposes.  */
5686   if (to == ADDL_REGS)
5687     to = GR_REGS;
5688   if (from == ADDL_REGS)
5689     from = GR_REGS;
5690
5691   /* All costs are symmetric, so reduce cases by putting the
5692      lower number class as the destination.  */
5693   if (from < to)
5694     {
5695       reg_class_t tmp = to;
5696       to = from, from = tmp;
5697     }
5698
5699   /* Moving from FR<->GR in XFmode must be more expensive than 2,
5700      so that we get secondary memory reloads.  Between FR_REGS,
5701      we have to make this at least as expensive as memory_move_cost
5702      to avoid spectacularly poor register class preferencing.  */
5703   if (mode == XFmode || mode == RFmode)
5704     {
5705       if (to != GR_REGS || from != GR_REGS)
5706         return memory_move_cost (mode, to, false);
5707       else
5708         return 3;
5709     }
5710
5711   switch (to)
5712     {
5713     case PR_REGS:
5714       /* Moving between PR registers takes two insns.  */
5715       if (from == PR_REGS)
5716         return 3;
5717       /* Moving between PR and anything but GR is impossible.  */
5718       if (from != GR_REGS)
5719         return memory_move_cost (mode, to, false);
5720       break;
5721
5722     case BR_REGS:
5723       /* Moving between BR and anything but GR is impossible.  */
5724       if (from != GR_REGS && from != GR_AND_BR_REGS)
5725         return memory_move_cost (mode, to, false);
5726       break;
5727
5728     case AR_I_REGS:
5729     case AR_M_REGS:
5730       /* Moving between AR and anything but GR is impossible.  */
5731       if (from != GR_REGS)
5732         return memory_move_cost (mode, to, false);
5733       break;
5734
5735     case GR_REGS:
5736     case FR_REGS:
5737     case FP_REGS:
5738     case GR_AND_FR_REGS:
5739     case GR_AND_BR_REGS:
5740     case ALL_REGS:
5741       break;
5742
5743     default:
5744       gcc_unreachable ();
5745     }
5746
5747   return 2;
5748 }
5749
5750 /* Calculate the cost of moving data of MODE from a register to or from
5751    memory.  */
5752
5753 static int
5754 ia64_memory_move_cost (machine_mode mode ATTRIBUTE_UNUSED,
5755                        reg_class_t rclass,
5756                        bool in ATTRIBUTE_UNUSED)
5757 {
5758   if (rclass == GENERAL_REGS
5759       || rclass == FR_REGS
5760       || rclass == FP_REGS
5761       || rclass == GR_AND_FR_REGS)
5762     return 4;
5763   else
5764     return 10;
5765 }
5766
5767 /* Implement TARGET_PREFERRED_RELOAD_CLASS.  Place additional restrictions
5768    on RCLASS to use when copying X into that class.  */
5769
5770 static reg_class_t
5771 ia64_preferred_reload_class (rtx x, reg_class_t rclass)
5772 {
5773   switch (rclass)
5774     {
5775     case FR_REGS:
5776     case FP_REGS:
5777       /* Don't allow volatile mem reloads into floating point registers.
5778          This is defined to force reload to choose the r/m case instead
5779          of the f/f case when reloading (set (reg fX) (mem/v)).  */
5780       if (MEM_P (x) && MEM_VOLATILE_P (x))
5781         return NO_REGS;
5782
5783       /* Force all unrecognized constants into the constant pool.  */
5784       if (CONSTANT_P (x))
5785         return NO_REGS;
5786       break;
5787
5788     case AR_M_REGS:
5789     case AR_I_REGS:
5790       if (!OBJECT_P (x))
5791         return NO_REGS;
5792       break;
5793
5794     default:
5795       break;
5796     }
5797
5798   return rclass;
5799 }
5800
5801 /* This function returns the register class required for a secondary
5802    register when copying between one of the registers in RCLASS, and X,
5803    using MODE.  A return value of NO_REGS means that no secondary register
5804    is required.  */
5805
5806 enum reg_class
5807 ia64_secondary_reload_class (enum reg_class rclass,
5808                              machine_mode mode ATTRIBUTE_UNUSED, rtx x)
5809 {
5810   int regno = -1;
5811
5812   if (GET_CODE (x) == REG || GET_CODE (x) == SUBREG)
5813     regno = true_regnum (x);
5814
5815   switch (rclass)
5816     {
5817     case BR_REGS:
5818     case AR_M_REGS:
5819     case AR_I_REGS:
5820       /* ??? BR<->BR register copies can happen due to a bad gcse/cse/global
5821          interaction.  We end up with two pseudos with overlapping lifetimes
5822          both of which are equiv to the same constant, and both which need
5823          to be in BR_REGS.  This seems to be a cse bug.  cse_basic_block_end
5824          changes depending on the path length, which means the qty_first_reg
5825          check in make_regs_eqv can give different answers at different times.
5826          At some point I'll probably need a reload_indi pattern to handle
5827          this.
5828
5829          We can also get GR_AND_FR_REGS to BR_REGS/AR_REGS copies, where we
5830          wound up with a FP register from GR_AND_FR_REGS.  Extend that to all
5831          non-general registers for good measure.  */
5832       if (regno >= 0 && ! GENERAL_REGNO_P (regno))
5833         return GR_REGS;
5834
5835       /* This is needed if a pseudo used as a call_operand gets spilled to a
5836          stack slot.  */
5837       if (GET_CODE (x) == MEM)
5838         return GR_REGS;
5839       break;
5840
5841     case FR_REGS:
5842     case FP_REGS:
5843       /* Need to go through general registers to get to other class regs.  */
5844       if (regno >= 0 && ! (FR_REGNO_P (regno) || GENERAL_REGNO_P (regno)))
5845         return GR_REGS;
5846
5847       /* This can happen when a paradoxical subreg is an operand to the
5848          muldi3 pattern.  */
5849       /* ??? This shouldn't be necessary after instruction scheduling is
5850          enabled, because paradoxical subregs are not accepted by
5851          register_operand when INSN_SCHEDULING is defined.  Or alternatively,
5852          stop the paradoxical subreg stupidity in the *_operand functions
5853          in recog.c.  */
5854       if (GET_CODE (x) == MEM
5855           && (GET_MODE (x) == SImode || GET_MODE (x) == HImode
5856               || GET_MODE (x) == QImode))
5857         return GR_REGS;
5858
5859       /* This can happen because of the ior/and/etc patterns that accept FP
5860          registers as operands.  If the third operand is a constant, then it
5861          needs to be reloaded into a FP register.  */
5862       if (GET_CODE (x) == CONST_INT)
5863         return GR_REGS;
5864
5865       /* This can happen because of register elimination in a muldi3 insn.
5866          E.g. `26107 * (unsigned long)&u'.  */
5867       if (GET_CODE (x) == PLUS)
5868         return GR_REGS;
5869       break;
5870
5871     case PR_REGS:
5872       /* ??? This happens if we cse/gcse a BImode value across a call,
5873          and the function has a nonlocal goto.  This is because global
5874          does not allocate call crossing pseudos to hard registers when
5875          crtl->has_nonlocal_goto is true.  This is relatively
5876          common for C++ programs that use exceptions.  To reproduce,
5877          return NO_REGS and compile libstdc++.  */
5878       if (GET_CODE (x) == MEM)
5879         return GR_REGS;
5880
5881       /* This can happen when we take a BImode subreg of a DImode value,
5882          and that DImode value winds up in some non-GR register.  */
5883       if (regno >= 0 && ! GENERAL_REGNO_P (regno) && ! PR_REGNO_P (regno))
5884         return GR_REGS;
5885       break;
5886
5887     default:
5888       break;
5889     }
5890
5891   return NO_REGS;
5892 }
5893
5894 \f
5895 /* Implement targetm.unspec_may_trap_p hook.  */
5896 static int
5897 ia64_unspec_may_trap_p (const_rtx x, unsigned flags)
5898 {
5899   switch (XINT (x, 1))
5900     {
5901     case UNSPEC_LDA:
5902     case UNSPEC_LDS:
5903     case UNSPEC_LDSA:
5904     case UNSPEC_LDCCLR:
5905     case UNSPEC_CHKACLR:
5906     case UNSPEC_CHKS:
5907       /* These unspecs are just wrappers.  */
5908       return may_trap_p_1 (XVECEXP (x, 0, 0), flags);
5909     }
5910
5911   return default_unspec_may_trap_p (x, flags);
5912 }
5913
5914 \f
5915 /* Parse the -mfixed-range= option string.  */
5916
5917 static void
5918 fix_range (const char *const_str)
5919 {
5920   int i, first, last;
5921   char *str, *dash, *comma;
5922
5923   /* str must be of the form REG1'-'REG2{,REG1'-'REG} where REG1 and
5924      REG2 are either register names or register numbers.  The effect
5925      of this option is to mark the registers in the range from REG1 to
5926      REG2 as ``fixed'' so they won't be used by the compiler.  This is
5927      used, e.g., to ensure that kernel mode code doesn't use f32-f127.  */
5928
5929   i = strlen (const_str);
5930   str = (char *) alloca (i + 1);
5931   memcpy (str, const_str, i + 1);
5932
5933   while (1)
5934     {
5935       dash = strchr (str, '-');
5936       if (!dash)
5937         {
5938           warning (0, "value of -mfixed-range must have form REG1-REG2");
5939           return;
5940         }
5941       *dash = '\0';
5942
5943       comma = strchr (dash + 1, ',');
5944       if (comma)
5945         *comma = '\0';
5946
5947       first = decode_reg_name (str);
5948       if (first < 0)
5949         {
5950           warning (0, "unknown register name: %s", str);
5951           return;
5952         }
5953
5954       last = decode_reg_name (dash + 1);
5955       if (last < 0)
5956         {
5957           warning (0, "unknown register name: %s", dash + 1);
5958           return;
5959         }
5960
5961       *dash = '-';
5962
5963       if (first > last)
5964         {
5965           warning (0, "%s-%s is an empty range", str, dash + 1);
5966           return;
5967         }
5968
5969       for (i = first; i <= last; ++i)
5970         fixed_regs[i] = call_used_regs[i] = 1;
5971
5972       if (!comma)
5973         break;
5974
5975       *comma = ',';
5976       str = comma + 1;
5977     }
5978 }
5979
5980 /* Implement TARGET_OPTION_OVERRIDE.  */
5981
5982 static void
5983 ia64_option_override (void)
5984 {
5985   unsigned int i;
5986   cl_deferred_option *opt;
5987   vec<cl_deferred_option> *v
5988     = (vec<cl_deferred_option> *) ia64_deferred_options;
5989
5990   if (v)
5991     FOR_EACH_VEC_ELT (*v, i, opt)
5992       {
5993         switch (opt->opt_index)
5994           {
5995           case OPT_mfixed_range_:
5996             fix_range (opt->arg);
5997             break;
5998
5999           default:
6000             gcc_unreachable ();
6001           }
6002       }
6003
6004   if (TARGET_AUTO_PIC)
6005     target_flags |= MASK_CONST_GP;
6006
6007   /* Numerous experiment shows that IRA based loop pressure
6008      calculation works better for RTL loop invariant motion on targets
6009      with enough (>= 32) registers.  It is an expensive optimization.
6010      So it is on only for peak performance.  */
6011   if (optimize >= 3)
6012     flag_ira_loop_pressure = 1;
6013
6014
6015   ia64_section_threshold = (global_options_set.x_g_switch_value
6016                             ? g_switch_value
6017                             : IA64_DEFAULT_GVALUE);
6018
6019   init_machine_status = ia64_init_machine_status;
6020
6021   if (align_functions <= 0)
6022     align_functions = 64;
6023   if (align_loops <= 0)
6024     align_loops = 32;
6025   if (TARGET_ABI_OPEN_VMS)
6026     flag_no_common = 1;
6027
6028   ia64_override_options_after_change();
6029 }
6030
6031 /* Implement targetm.override_options_after_change.  */
6032
6033 static void
6034 ia64_override_options_after_change (void)
6035 {
6036   if (optimize >= 3
6037       && !global_options_set.x_flag_selective_scheduling
6038       && !global_options_set.x_flag_selective_scheduling2)
6039     {
6040       flag_selective_scheduling2 = 1;
6041       flag_sel_sched_pipelining = 1;
6042     }
6043   if (mflag_sched_control_spec == 2)
6044     {
6045       /* Control speculation is on by default for the selective scheduler,
6046          but not for the Haifa scheduler.  */
6047       mflag_sched_control_spec = flag_selective_scheduling2 ? 1 : 0;
6048     }
6049   if (flag_sel_sched_pipelining && flag_auto_inc_dec)
6050     {
6051       /* FIXME: remove this when we'd implement breaking autoinsns as
6052          a transformation.  */
6053       flag_auto_inc_dec = 0;
6054     }
6055 }
6056
6057 /* Initialize the record of emitted frame related registers.  */
6058
6059 void ia64_init_expanders (void)
6060 {
6061   memset (&emitted_frame_related_regs, 0, sizeof (emitted_frame_related_regs));
6062 }
6063
6064 static struct machine_function *
6065 ia64_init_machine_status (void)
6066 {
6067   return ggc_cleared_alloc<machine_function> ();
6068 }
6069 \f
6070 static enum attr_itanium_class ia64_safe_itanium_class (rtx_insn *);
6071 static enum attr_type ia64_safe_type (rtx_insn *);
6072
6073 static enum attr_itanium_class
6074 ia64_safe_itanium_class (rtx_insn *insn)
6075 {
6076   if (recog_memoized (insn) >= 0)
6077     return get_attr_itanium_class (insn);
6078   else if (DEBUG_INSN_P (insn))
6079     return ITANIUM_CLASS_IGNORE;
6080   else
6081     return ITANIUM_CLASS_UNKNOWN;
6082 }
6083
6084 static enum attr_type
6085 ia64_safe_type (rtx_insn *insn)
6086 {
6087   if (recog_memoized (insn) >= 0)
6088     return get_attr_type (insn);
6089   else
6090     return TYPE_UNKNOWN;
6091 }
6092 \f
6093 /* The following collection of routines emit instruction group stop bits as
6094    necessary to avoid dependencies.  */
6095
6096 /* Need to track some additional registers as far as serialization is
6097    concerned so we can properly handle br.call and br.ret.  We could
6098    make these registers visible to gcc, but since these registers are
6099    never explicitly used in gcc generated code, it seems wasteful to
6100    do so (plus it would make the call and return patterns needlessly
6101    complex).  */
6102 #define REG_RP          (BR_REG (0))
6103 #define REG_AR_CFM      (FIRST_PSEUDO_REGISTER + 1)
6104 /* This is used for volatile asms which may require a stop bit immediately
6105    before and after them.  */
6106 #define REG_VOLATILE    (FIRST_PSEUDO_REGISTER + 2)
6107 #define AR_UNAT_BIT_0   (FIRST_PSEUDO_REGISTER + 3)
6108 #define NUM_REGS        (AR_UNAT_BIT_0 + 64)
6109
6110 /* For each register, we keep track of how it has been written in the
6111    current instruction group.
6112
6113    If a register is written unconditionally (no qualifying predicate),
6114    WRITE_COUNT is set to 2 and FIRST_PRED is ignored.
6115
6116    If a register is written if its qualifying predicate P is true, we
6117    set WRITE_COUNT to 1 and FIRST_PRED to P.  Later on, the same register
6118    may be written again by the complement of P (P^1) and when this happens,
6119    WRITE_COUNT gets set to 2.
6120
6121    The result of this is that whenever an insn attempts to write a register
6122    whose WRITE_COUNT is two, we need to issue an insn group barrier first.
6123
6124    If a predicate register is written by a floating-point insn, we set
6125    WRITTEN_BY_FP to true.
6126
6127    If a predicate register is written by an AND.ORCM we set WRITTEN_BY_AND
6128    to true; if it was written by an OR.ANDCM we set WRITTEN_BY_OR to true.  */
6129
6130 #if GCC_VERSION >= 4000
6131 #define RWS_FIELD_TYPE __extension__ unsigned short
6132 #else
6133 #define RWS_FIELD_TYPE unsigned int
6134 #endif
6135 struct reg_write_state
6136 {
6137   RWS_FIELD_TYPE write_count : 2;
6138   RWS_FIELD_TYPE first_pred : 10;
6139   RWS_FIELD_TYPE written_by_fp : 1;
6140   RWS_FIELD_TYPE written_by_and : 1;
6141   RWS_FIELD_TYPE written_by_or : 1;
6142 };
6143
6144 /* Cumulative info for the current instruction group.  */
6145 struct reg_write_state rws_sum[NUM_REGS];
6146 #ifdef ENABLE_CHECKING
6147 /* Bitmap whether a register has been written in the current insn.  */
6148 HARD_REG_ELT_TYPE rws_insn[(NUM_REGS + HOST_BITS_PER_WIDEST_FAST_INT - 1)
6149                            / HOST_BITS_PER_WIDEST_FAST_INT];
6150
6151 static inline void
6152 rws_insn_set (int regno)
6153 {
6154   gcc_assert (!TEST_HARD_REG_BIT (rws_insn, regno));
6155   SET_HARD_REG_BIT (rws_insn, regno);
6156 }
6157
6158 static inline int
6159 rws_insn_test (int regno)
6160 {
6161   return TEST_HARD_REG_BIT (rws_insn, regno);
6162 }
6163 #else
6164 /* When not checking, track just REG_AR_CFM and REG_VOLATILE.  */
6165 unsigned char rws_insn[2];
6166
6167 static inline void
6168 rws_insn_set (int regno)
6169 {
6170   if (regno == REG_AR_CFM)
6171     rws_insn[0] = 1;
6172   else if (regno == REG_VOLATILE)
6173     rws_insn[1] = 1;
6174 }
6175
6176 static inline int
6177 rws_insn_test (int regno)
6178 {
6179   if (regno == REG_AR_CFM)
6180     return rws_insn[0];
6181   if (regno == REG_VOLATILE)
6182     return rws_insn[1];
6183   return 0;
6184 }
6185 #endif
6186
6187 /* Indicates whether this is the first instruction after a stop bit,
6188    in which case we don't need another stop bit.  Without this,
6189    ia64_variable_issue will die when scheduling an alloc.  */
6190 static int first_instruction;
6191
6192 /* Misc flags needed to compute RAW/WAW dependencies while we are traversing
6193    RTL for one instruction.  */
6194 struct reg_flags
6195 {
6196   unsigned int is_write : 1;    /* Is register being written?  */
6197   unsigned int is_fp : 1;       /* Is register used as part of an fp op?  */
6198   unsigned int is_branch : 1;   /* Is register used as part of a branch?  */
6199   unsigned int is_and : 1;      /* Is register used as part of and.orcm?  */
6200   unsigned int is_or : 1;       /* Is register used as part of or.andcm?  */
6201   unsigned int is_sibcall : 1;  /* Is this a sibling or normal call?  */
6202 };
6203
6204 static void rws_update (int, struct reg_flags, int);
6205 static int rws_access_regno (int, struct reg_flags, int);
6206 static int rws_access_reg (rtx, struct reg_flags, int);
6207 static void update_set_flags (rtx, struct reg_flags *);
6208 static int set_src_needs_barrier (rtx, struct reg_flags, int);
6209 static int rtx_needs_barrier (rtx, struct reg_flags, int);
6210 static void init_insn_group_barriers (void);
6211 static int group_barrier_needed (rtx_insn *);
6212 static int safe_group_barrier_needed (rtx_insn *);
6213 static int in_safe_group_barrier;
6214
6215 /* Update *RWS for REGNO, which is being written by the current instruction,
6216    with predicate PRED, and associated register flags in FLAGS.  */
6217
6218 static void
6219 rws_update (int regno, struct reg_flags flags, int pred)
6220 {
6221   if (pred)
6222     rws_sum[regno].write_count++;
6223   else
6224     rws_sum[regno].write_count = 2;
6225   rws_sum[regno].written_by_fp |= flags.is_fp;
6226   /* ??? Not tracking and/or across differing predicates.  */
6227   rws_sum[regno].written_by_and = flags.is_and;
6228   rws_sum[regno].written_by_or = flags.is_or;
6229   rws_sum[regno].first_pred = pred;
6230 }
6231
6232 /* Handle an access to register REGNO of type FLAGS using predicate register
6233    PRED.  Update rws_sum array.  Return 1 if this access creates
6234    a dependency with an earlier instruction in the same group.  */
6235
6236 static int
6237 rws_access_regno (int regno, struct reg_flags flags, int pred)
6238 {
6239   int need_barrier = 0;
6240
6241   gcc_assert (regno < NUM_REGS);
6242
6243   if (! PR_REGNO_P (regno))
6244     flags.is_and = flags.is_or = 0;
6245
6246   if (flags.is_write)
6247     {
6248       int write_count;
6249
6250       rws_insn_set (regno);
6251       write_count = rws_sum[regno].write_count;
6252
6253       switch (write_count)
6254         {
6255         case 0:
6256           /* The register has not been written yet.  */
6257           if (!in_safe_group_barrier)
6258             rws_update (regno, flags, pred);
6259           break;
6260
6261         case 1:
6262           /* The register has been written via a predicate.  Treat
6263              it like a unconditional write and do not try to check
6264              for complementary pred reg in earlier write.  */
6265           if (flags.is_and && rws_sum[regno].written_by_and)
6266             ;
6267           else if (flags.is_or && rws_sum[regno].written_by_or)
6268             ;
6269           else
6270             need_barrier = 1;
6271           if (!in_safe_group_barrier)
6272             rws_update (regno, flags, pred);
6273           break;
6274
6275         case 2:
6276           /* The register has been unconditionally written already.  We
6277              need a barrier.  */
6278           if (flags.is_and && rws_sum[regno].written_by_and)
6279             ;
6280           else if (flags.is_or && rws_sum[regno].written_by_or)
6281             ;
6282           else
6283             need_barrier = 1;
6284           if (!in_safe_group_barrier)
6285             {
6286               rws_sum[regno].written_by_and = flags.is_and;
6287               rws_sum[regno].written_by_or = flags.is_or;
6288             }
6289           break;
6290
6291         default:
6292           gcc_unreachable ();
6293         }
6294     }
6295   else
6296     {
6297       if (flags.is_branch)
6298         {
6299           /* Branches have several RAW exceptions that allow to avoid
6300              barriers.  */
6301
6302           if (REGNO_REG_CLASS (regno) == BR_REGS || regno == AR_PFS_REGNUM)
6303             /* RAW dependencies on branch regs are permissible as long
6304                as the writer is a non-branch instruction.  Since we
6305                never generate code that uses a branch register written
6306                by a branch instruction, handling this case is
6307                easy.  */
6308             return 0;
6309
6310           if (REGNO_REG_CLASS (regno) == PR_REGS
6311               && ! rws_sum[regno].written_by_fp)
6312             /* The predicates of a branch are available within the
6313                same insn group as long as the predicate was written by
6314                something other than a floating-point instruction.  */
6315             return 0;
6316         }
6317
6318       if (flags.is_and && rws_sum[regno].written_by_and)
6319         return 0;
6320       if (flags.is_or && rws_sum[regno].written_by_or)
6321         return 0;
6322
6323       switch (rws_sum[regno].write_count)
6324         {
6325         case 0:
6326           /* The register has not been written yet.  */
6327           break;
6328
6329         case 1:
6330           /* The register has been written via a predicate, assume we
6331              need a barrier (don't check for complementary regs).  */
6332           need_barrier = 1;
6333           break;
6334
6335         case 2:
6336           /* The register has been unconditionally written already.  We
6337              need a barrier.  */
6338           need_barrier = 1;
6339           break;
6340
6341         default:
6342           gcc_unreachable ();
6343         }
6344     }
6345
6346   return need_barrier;
6347 }
6348
6349 static int
6350 rws_access_reg (rtx reg, struct reg_flags flags, int pred)
6351 {
6352   int regno = REGNO (reg);
6353   int n = HARD_REGNO_NREGS (REGNO (reg), GET_MODE (reg));
6354
6355   if (n == 1)
6356     return rws_access_regno (regno, flags, pred);
6357   else
6358     {
6359       int need_barrier = 0;
6360       while (--n >= 0)
6361         need_barrier |= rws_access_regno (regno + n, flags, pred);
6362       return need_barrier;
6363     }
6364 }
6365
6366 /* Examine X, which is a SET rtx, and update the flags, the predicate, and
6367    the condition, stored in *PFLAGS, *PPRED and *PCOND.  */
6368
6369 static void
6370 update_set_flags (rtx x, struct reg_flags *pflags)
6371 {
6372   rtx src = SET_SRC (x);
6373
6374   switch (GET_CODE (src))
6375     {
6376     case CALL:
6377       return;
6378
6379     case IF_THEN_ELSE:
6380       /* There are four cases here:
6381          (1) The destination is (pc), in which case this is a branch,
6382          nothing here applies.
6383          (2) The destination is ar.lc, in which case this is a
6384          doloop_end_internal,
6385          (3) The destination is an fp register, in which case this is
6386          an fselect instruction.
6387          (4) The condition has (unspec [(reg)] UNSPEC_LDC), in which case
6388          this is a check load.
6389          In all cases, nothing we do in this function applies.  */
6390       return;
6391
6392     default:
6393       if (COMPARISON_P (src)
6394           && SCALAR_FLOAT_MODE_P (GET_MODE (XEXP (src, 0))))
6395         /* Set pflags->is_fp to 1 so that we know we're dealing
6396            with a floating point comparison when processing the
6397            destination of the SET.  */
6398         pflags->is_fp = 1;
6399
6400       /* Discover if this is a parallel comparison.  We only handle
6401          and.orcm and or.andcm at present, since we must retain a
6402          strict inverse on the predicate pair.  */
6403       else if (GET_CODE (src) == AND)
6404         pflags->is_and = 1;
6405       else if (GET_CODE (src) == IOR)
6406         pflags->is_or = 1;
6407
6408       break;
6409     }
6410 }
6411
6412 /* Subroutine of rtx_needs_barrier; this function determines whether the
6413    source of a given SET rtx found in X needs a barrier.  FLAGS and PRED
6414    are as in rtx_needs_barrier.  COND is an rtx that holds the condition
6415    for this insn.  */
6416
6417 static int
6418 set_src_needs_barrier (rtx x, struct reg_flags flags, int pred)
6419 {
6420   int need_barrier = 0;
6421   rtx dst;
6422   rtx src = SET_SRC (x);
6423
6424   if (GET_CODE (src) == CALL)
6425     /* We don't need to worry about the result registers that
6426        get written by subroutine call.  */
6427     return rtx_needs_barrier (src, flags, pred);
6428   else if (SET_DEST (x) == pc_rtx)
6429     {
6430       /* X is a conditional branch.  */
6431       /* ??? This seems redundant, as the caller sets this bit for
6432          all JUMP_INSNs.  */
6433       if (!ia64_spec_check_src_p (src))
6434         flags.is_branch = 1;
6435       return rtx_needs_barrier (src, flags, pred);
6436     }
6437
6438   if (ia64_spec_check_src_p (src))
6439     /* Avoid checking one register twice (in condition
6440        and in 'then' section) for ldc pattern.  */
6441     {
6442       gcc_assert (REG_P (XEXP (src, 2)));
6443       need_barrier = rtx_needs_barrier (XEXP (src, 2), flags, pred);
6444
6445       /* We process MEM below.  */
6446       src = XEXP (src, 1);
6447     }
6448
6449   need_barrier |= rtx_needs_barrier (src, flags, pred);
6450
6451   dst = SET_DEST (x);
6452   if (GET_CODE (dst) == ZERO_EXTRACT)
6453     {
6454       need_barrier |= rtx_needs_barrier (XEXP (dst, 1), flags, pred);
6455       need_barrier |= rtx_needs_barrier (XEXP (dst, 2), flags, pred);
6456     }
6457   return need_barrier;
6458 }
6459
6460 /* Handle an access to rtx X of type FLAGS using predicate register
6461    PRED.  Return 1 if this access creates a dependency with an earlier
6462    instruction in the same group.  */
6463
6464 static int
6465 rtx_needs_barrier (rtx x, struct reg_flags flags, int pred)
6466 {
6467   int i, j;
6468   int is_complemented = 0;
6469   int need_barrier = 0;
6470   const char *format_ptr;
6471   struct reg_flags new_flags;
6472   rtx cond;
6473
6474   if (! x)
6475     return 0;
6476
6477   new_flags = flags;
6478
6479   switch (GET_CODE (x))
6480     {
6481     case SET:
6482       update_set_flags (x, &new_flags);
6483       need_barrier = set_src_needs_barrier (x, new_flags, pred);
6484       if (GET_CODE (SET_SRC (x)) != CALL)
6485         {
6486           new_flags.is_write = 1;
6487           need_barrier |= rtx_needs_barrier (SET_DEST (x), new_flags, pred);
6488         }
6489       break;
6490
6491     case CALL:
6492       new_flags.is_write = 0;
6493       need_barrier |= rws_access_regno (AR_EC_REGNUM, new_flags, pred);
6494
6495       /* Avoid multiple register writes, in case this is a pattern with
6496          multiple CALL rtx.  This avoids a failure in rws_access_reg.  */
6497       if (! flags.is_sibcall && ! rws_insn_test (REG_AR_CFM))
6498         {
6499           new_flags.is_write = 1;
6500           need_barrier |= rws_access_regno (REG_RP, new_flags, pred);
6501           need_barrier |= rws_access_regno (AR_PFS_REGNUM, new_flags, pred);
6502           need_barrier |= rws_access_regno (REG_AR_CFM, new_flags, pred);
6503         }
6504       break;
6505
6506     case COND_EXEC:
6507       /* X is a predicated instruction.  */
6508
6509       cond = COND_EXEC_TEST (x);
6510       gcc_assert (!pred);
6511       need_barrier = rtx_needs_barrier (cond, flags, 0);
6512
6513       if (GET_CODE (cond) == EQ)
6514         is_complemented = 1;
6515       cond = XEXP (cond, 0);
6516       gcc_assert (GET_CODE (cond) == REG
6517                   && REGNO_REG_CLASS (REGNO (cond)) == PR_REGS);
6518       pred = REGNO (cond);
6519       if (is_complemented)
6520         ++pred;
6521
6522       need_barrier |= rtx_needs_barrier (COND_EXEC_CODE (x), flags, pred);
6523       return need_barrier;
6524
6525     case CLOBBER:
6526     case USE:
6527       /* Clobber & use are for earlier compiler-phases only.  */
6528       break;
6529
6530     case ASM_OPERANDS:
6531     case ASM_INPUT:
6532       /* We always emit stop bits for traditional asms.  We emit stop bits
6533          for volatile extended asms if TARGET_VOL_ASM_STOP is true.  */
6534       if (GET_CODE (x) != ASM_OPERANDS
6535           || (MEM_VOLATILE_P (x) && TARGET_VOL_ASM_STOP))
6536         {
6537           /* Avoid writing the register multiple times if we have multiple
6538              asm outputs.  This avoids a failure in rws_access_reg.  */
6539           if (! rws_insn_test (REG_VOLATILE))
6540             {
6541               new_flags.is_write = 1;
6542               rws_access_regno (REG_VOLATILE, new_flags, pred);
6543             }
6544           return 1;
6545         }
6546
6547       /* For all ASM_OPERANDS, we must traverse the vector of input operands.
6548          We cannot just fall through here since then we would be confused
6549          by the ASM_INPUT rtx inside ASM_OPERANDS, which do not indicate
6550          traditional asms unlike their normal usage.  */
6551
6552       for (i = ASM_OPERANDS_INPUT_LENGTH (x) - 1; i >= 0; --i)
6553         if (rtx_needs_barrier (ASM_OPERANDS_INPUT (x, i), flags, pred))
6554           need_barrier = 1;
6555       break;
6556
6557     case PARALLEL:
6558       for (i = XVECLEN (x, 0) - 1; i >= 0; --i)
6559         {
6560           rtx pat = XVECEXP (x, 0, i);
6561           switch (GET_CODE (pat))
6562             {
6563             case SET:
6564               update_set_flags (pat, &new_flags);
6565               need_barrier |= set_src_needs_barrier (pat, new_flags, pred);
6566               break;
6567
6568             case USE:
6569             case CALL:
6570             case ASM_OPERANDS:
6571               need_barrier |= rtx_needs_barrier (pat, flags, pred);
6572               break;
6573
6574             case CLOBBER:
6575               if (REG_P (XEXP (pat, 0))
6576                   && extract_asm_operands (x) != NULL_RTX
6577                   && REGNO (XEXP (pat, 0)) != AR_UNAT_REGNUM)
6578                 {
6579                   new_flags.is_write = 1;
6580                   need_barrier |= rtx_needs_barrier (XEXP (pat, 0),
6581                                                      new_flags, pred);
6582                   new_flags = flags;
6583                 }
6584               break;
6585
6586             case RETURN:
6587               break;
6588
6589             default:
6590               gcc_unreachable ();
6591             }
6592         }
6593       for (i = XVECLEN (x, 0) - 1; i >= 0; --i)
6594         {
6595           rtx pat = XVECEXP (x, 0, i);
6596           if (GET_CODE (pat) == SET)
6597             {
6598               if (GET_CODE (SET_SRC (pat)) != CALL)
6599                 {
6600                   new_flags.is_write = 1;
6601                   need_barrier |= rtx_needs_barrier (SET_DEST (pat), new_flags,
6602                                                      pred);
6603                 }
6604             }
6605           else if (GET_CODE (pat) == CLOBBER || GET_CODE (pat) == RETURN)
6606             need_barrier |= rtx_needs_barrier (pat, flags, pred);
6607         }
6608       break;
6609
6610     case SUBREG:
6611       need_barrier |= rtx_needs_barrier (SUBREG_REG (x), flags, pred);
6612       break;
6613     case REG:
6614       if (REGNO (x) == AR_UNAT_REGNUM)
6615         {
6616           for (i = 0; i < 64; ++i)
6617             need_barrier |= rws_access_regno (AR_UNAT_BIT_0 + i, flags, pred);
6618         }
6619       else
6620         need_barrier = rws_access_reg (x, flags, pred);
6621       break;
6622
6623     case MEM:
6624       /* Find the regs used in memory address computation.  */
6625       new_flags.is_write = 0;
6626       need_barrier = rtx_needs_barrier (XEXP (x, 0), new_flags, pred);
6627       break;
6628
6629     case CONST_INT:   case CONST_DOUBLE:  case CONST_VECTOR:
6630     case SYMBOL_REF:  case LABEL_REF:     case CONST:
6631       break;
6632
6633       /* Operators with side-effects.  */
6634     case POST_INC:    case POST_DEC:
6635       gcc_assert (GET_CODE (XEXP (x, 0)) == REG);
6636
6637       new_flags.is_write = 0;
6638       need_barrier  = rws_access_reg (XEXP (x, 0), new_flags, pred);
6639       new_flags.is_write = 1;
6640       need_barrier |= rws_access_reg (XEXP (x, 0), new_flags, pred);
6641       break;
6642
6643     case POST_MODIFY:
6644       gcc_assert (GET_CODE (XEXP (x, 0)) == REG);
6645
6646       new_flags.is_write = 0;
6647       need_barrier  = rws_access_reg (XEXP (x, 0), new_flags, pred);
6648       need_barrier |= rtx_needs_barrier (XEXP (x, 1), new_flags, pred);
6649       new_flags.is_write = 1;
6650       need_barrier |= rws_access_reg (XEXP (x, 0), new_flags, pred);
6651       break;
6652
6653       /* Handle common unary and binary ops for efficiency.  */
6654     case COMPARE:  case PLUS:    case MINUS:   case MULT:      case DIV:
6655     case MOD:      case UDIV:    case UMOD:    case AND:       case IOR:
6656     case XOR:      case ASHIFT:  case ROTATE:  case ASHIFTRT:  case LSHIFTRT:
6657     case ROTATERT: case SMIN:    case SMAX:    case UMIN:      case UMAX:
6658     case NE:       case EQ:      case GE:      case GT:        case LE:
6659     case LT:       case GEU:     case GTU:     case LEU:       case LTU:
6660       need_barrier = rtx_needs_barrier (XEXP (x, 0), new_flags, pred);
6661       need_barrier |= rtx_needs_barrier (XEXP (x, 1), new_flags, pred);
6662       break;
6663
6664     case NEG:      case NOT:            case SIGN_EXTEND:     case ZERO_EXTEND:
6665     case TRUNCATE: case FLOAT_EXTEND:   case FLOAT_TRUNCATE:  case FLOAT:
6666     case FIX:      case UNSIGNED_FLOAT: case UNSIGNED_FIX:    case ABS:
6667     case SQRT:     case FFS:            case POPCOUNT:
6668       need_barrier = rtx_needs_barrier (XEXP (x, 0), flags, pred);
6669       break;
6670
6671     case VEC_SELECT:
6672       /* VEC_SELECT's second argument is a PARALLEL with integers that
6673          describe the elements selected.  On ia64, those integers are
6674          always constants.  Avoid walking the PARALLEL so that we don't
6675          get confused with "normal" parallels and then die.  */
6676       need_barrier = rtx_needs_barrier (XEXP (x, 0), flags, pred);
6677       break;
6678
6679     case UNSPEC:
6680       switch (XINT (x, 1))
6681         {
6682         case UNSPEC_LTOFF_DTPMOD:
6683         case UNSPEC_LTOFF_DTPREL:
6684         case UNSPEC_DTPREL:
6685         case UNSPEC_LTOFF_TPREL:
6686         case UNSPEC_TPREL:
6687         case UNSPEC_PRED_REL_MUTEX:
6688         case UNSPEC_PIC_CALL:
6689         case UNSPEC_MF:
6690         case UNSPEC_FETCHADD_ACQ:
6691         case UNSPEC_FETCHADD_REL:
6692         case UNSPEC_BSP_VALUE:
6693         case UNSPEC_FLUSHRS:
6694         case UNSPEC_BUNDLE_SELECTOR:
6695           break;
6696
6697         case UNSPEC_GR_SPILL:
6698         case UNSPEC_GR_RESTORE:
6699           {
6700             HOST_WIDE_INT offset = INTVAL (XVECEXP (x, 0, 1));
6701             HOST_WIDE_INT bit = (offset >> 3) & 63;
6702
6703             need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6704             new_flags.is_write = (XINT (x, 1) == UNSPEC_GR_SPILL);
6705             need_barrier |= rws_access_regno (AR_UNAT_BIT_0 + bit,
6706                                               new_flags, pred);
6707             break;
6708           }
6709
6710         case UNSPEC_FR_SPILL:
6711         case UNSPEC_FR_RESTORE:
6712         case UNSPEC_GETF_EXP:
6713         case UNSPEC_SETF_EXP:
6714         case UNSPEC_ADDP4:
6715         case UNSPEC_FR_SQRT_RECIP_APPROX:
6716         case UNSPEC_FR_SQRT_RECIP_APPROX_RES:
6717         case UNSPEC_LDA:
6718         case UNSPEC_LDS:
6719         case UNSPEC_LDS_A:
6720         case UNSPEC_LDSA:
6721         case UNSPEC_CHKACLR:
6722         case UNSPEC_CHKS:
6723           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6724           break;
6725
6726         case UNSPEC_FR_RECIP_APPROX:
6727         case UNSPEC_SHRP:
6728         case UNSPEC_COPYSIGN:
6729         case UNSPEC_FR_RECIP_APPROX_RES:
6730           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6731           need_barrier |= rtx_needs_barrier (XVECEXP (x, 0, 1), flags, pred);
6732           break;
6733
6734         case UNSPEC_CMPXCHG_ACQ:
6735         case UNSPEC_CMPXCHG_REL:
6736           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 1), flags, pred);
6737           need_barrier |= rtx_needs_barrier (XVECEXP (x, 0, 2), flags, pred);
6738           break;
6739
6740         default:
6741           gcc_unreachable ();
6742         }
6743       break;
6744
6745     case UNSPEC_VOLATILE:
6746       switch (XINT (x, 1))
6747         {
6748         case UNSPECV_ALLOC:
6749           /* Alloc must always be the first instruction of a group.
6750              We force this by always returning true.  */
6751           /* ??? We might get better scheduling if we explicitly check for
6752              input/local/output register dependencies, and modify the
6753              scheduler so that alloc is always reordered to the start of
6754              the current group.  We could then eliminate all of the
6755              first_instruction code.  */
6756           rws_access_regno (AR_PFS_REGNUM, flags, pred);
6757
6758           new_flags.is_write = 1;
6759           rws_access_regno (REG_AR_CFM, new_flags, pred);
6760           return 1;
6761
6762         case UNSPECV_SET_BSP:
6763         case UNSPECV_PROBE_STACK_RANGE:
6764           need_barrier = 1;
6765           break;
6766
6767         case UNSPECV_BLOCKAGE:
6768         case UNSPECV_INSN_GROUP_BARRIER:
6769         case UNSPECV_BREAK:
6770         case UNSPECV_PSAC_ALL:
6771         case UNSPECV_PSAC_NORMAL:
6772           return 0;
6773
6774         case UNSPECV_PROBE_STACK_ADDRESS:
6775           need_barrier = rtx_needs_barrier (XVECEXP (x, 0, 0), flags, pred);
6776           break;
6777
6778         default:
6779           gcc_unreachable ();
6780         }
6781       break;
6782
6783     case RETURN:
6784       new_flags.is_write = 0;
6785       need_barrier  = rws_access_regno (REG_RP, flags, pred);
6786       need_barrier |= rws_access_regno (AR_PFS_REGNUM, flags, pred);
6787
6788       new_flags.is_write = 1;
6789       need_barrier |= rws_access_regno (AR_EC_REGNUM, new_flags, pred);
6790       need_barrier |= rws_access_regno (REG_AR_CFM, new_flags, pred);
6791       break;
6792
6793     default:
6794       format_ptr = GET_RTX_FORMAT (GET_CODE (x));
6795       for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
6796         switch (format_ptr[i])
6797           {
6798           case '0':     /* unused field */
6799           case 'i':     /* integer */
6800           case 'n':     /* note */
6801           case 'w':     /* wide integer */
6802           case 's':     /* pointer to string */
6803           case 'S':     /* optional pointer to string */
6804             break;
6805
6806           case 'e':
6807             if (rtx_needs_barrier (XEXP (x, i), flags, pred))
6808               need_barrier = 1;
6809             break;
6810
6811           case 'E':
6812             for (j = XVECLEN (x, i) - 1; j >= 0; --j)
6813               if (rtx_needs_barrier (XVECEXP (x, i, j), flags, pred))
6814                 need_barrier = 1;
6815             break;
6816
6817           default:
6818             gcc_unreachable ();
6819           }
6820       break;
6821     }
6822   return need_barrier;
6823 }
6824
6825 /* Clear out the state for group_barrier_needed at the start of a
6826    sequence of insns.  */
6827
6828 static void
6829 init_insn_group_barriers (void)
6830 {
6831   memset (rws_sum, 0, sizeof (rws_sum));
6832   first_instruction = 1;
6833 }
6834
6835 /* Given the current state, determine whether a group barrier (a stop bit) is
6836    necessary before INSN.  Return nonzero if so.  This modifies the state to
6837    include the effects of INSN as a side-effect.  */
6838
6839 static int
6840 group_barrier_needed (rtx_insn *insn)
6841 {
6842   rtx pat;
6843   int need_barrier = 0;
6844   struct reg_flags flags;
6845
6846   memset (&flags, 0, sizeof (flags));
6847   switch (GET_CODE (insn))
6848     {
6849     case NOTE:
6850     case DEBUG_INSN:
6851       break;
6852
6853     case BARRIER:
6854       /* A barrier doesn't imply an instruction group boundary.  */
6855       break;
6856
6857     case CODE_LABEL:
6858       memset (rws_insn, 0, sizeof (rws_insn));
6859       return 1;
6860
6861     case CALL_INSN:
6862       flags.is_branch = 1;
6863       flags.is_sibcall = SIBLING_CALL_P (insn);
6864       memset (rws_insn, 0, sizeof (rws_insn));
6865
6866       /* Don't bundle a call following another call.  */
6867       if ((pat = prev_active_insn (insn)) && CALL_P (pat))
6868         {
6869           need_barrier = 1;
6870           break;
6871         }
6872
6873       need_barrier = rtx_needs_barrier (PATTERN (insn), flags, 0);
6874       break;
6875
6876     case JUMP_INSN:
6877       if (!ia64_spec_check_p (insn))
6878         flags.is_branch = 1;
6879
6880       /* Don't bundle a jump following a call.  */
6881       if ((pat = prev_active_insn (insn)) && CALL_P (pat))
6882         {
6883           need_barrier = 1;
6884           break;
6885         }
6886       /* FALLTHRU */
6887
6888     case INSN:
6889       if (GET_CODE (PATTERN (insn)) == USE
6890           || GET_CODE (PATTERN (insn)) == CLOBBER)
6891         /* Don't care about USE and CLOBBER "insns"---those are used to
6892            indicate to the optimizer that it shouldn't get rid of
6893            certain operations.  */
6894         break;
6895
6896       pat = PATTERN (insn);
6897
6898       /* Ug.  Hack hacks hacked elsewhere.  */
6899       switch (recog_memoized (insn))
6900         {
6901           /* We play dependency tricks with the epilogue in order
6902              to get proper schedules.  Undo this for dv analysis.  */
6903         case CODE_FOR_epilogue_deallocate_stack:
6904         case CODE_FOR_prologue_allocate_stack:
6905           pat = XVECEXP (pat, 0, 0);
6906           break;
6907
6908           /* The pattern we use for br.cloop confuses the code above.
6909              The second element of the vector is representative.  */
6910         case CODE_FOR_doloop_end_internal:
6911           pat = XVECEXP (pat, 0, 1);
6912           break;
6913
6914           /* Doesn't generate code.  */
6915         case CODE_FOR_pred_rel_mutex:
6916         case CODE_FOR_prologue_use:
6917           return 0;
6918
6919         default:
6920           break;
6921         }
6922
6923       memset (rws_insn, 0, sizeof (rws_insn));
6924       need_barrier = rtx_needs_barrier (pat, flags, 0);
6925
6926       /* Check to see if the previous instruction was a volatile
6927          asm.  */
6928       if (! need_barrier)
6929         need_barrier = rws_access_regno (REG_VOLATILE, flags, 0);
6930
6931       break;
6932
6933     default:
6934       gcc_unreachable ();
6935     }
6936
6937   if (first_instruction && important_for_bundling_p (insn))
6938     {
6939       need_barrier = 0;
6940       first_instruction = 0;
6941     }
6942
6943   return need_barrier;
6944 }
6945
6946 /* Like group_barrier_needed, but do not clobber the current state.  */
6947
6948 static int
6949 safe_group_barrier_needed (rtx_insn *insn)
6950 {
6951   int saved_first_instruction;
6952   int t;
6953
6954   saved_first_instruction = first_instruction;
6955   in_safe_group_barrier = 1;
6956
6957   t = group_barrier_needed (insn);
6958
6959   first_instruction = saved_first_instruction;
6960   in_safe_group_barrier = 0;
6961
6962   return t;
6963 }
6964
6965 /* Scan the current function and insert stop bits as necessary to
6966    eliminate dependencies.  This function assumes that a final
6967    instruction scheduling pass has been run which has already
6968    inserted most of the necessary stop bits.  This function only
6969    inserts new ones at basic block boundaries, since these are
6970    invisible to the scheduler.  */
6971
6972 static void
6973 emit_insn_group_barriers (FILE *dump)
6974 {
6975   rtx_insn *insn;
6976   rtx_insn *last_label = 0;
6977   int insns_since_last_label = 0;
6978
6979   init_insn_group_barriers ();
6980
6981   for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
6982     {
6983       if (LABEL_P (insn))
6984         {
6985           if (insns_since_last_label)
6986             last_label = insn;
6987           insns_since_last_label = 0;
6988         }
6989       else if (NOTE_P (insn)
6990                && NOTE_KIND (insn) == NOTE_INSN_BASIC_BLOCK)
6991         {
6992           if (insns_since_last_label)
6993             last_label = insn;
6994           insns_since_last_label = 0;
6995         }
6996       else if (NONJUMP_INSN_P (insn)
6997                && GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
6998                && XINT (PATTERN (insn), 1) == UNSPECV_INSN_GROUP_BARRIER)
6999         {
7000           init_insn_group_barriers ();
7001           last_label = 0;
7002         }
7003       else if (NONDEBUG_INSN_P (insn))
7004         {
7005           insns_since_last_label = 1;
7006
7007           if (group_barrier_needed (insn))
7008             {
7009               if (last_label)
7010                 {
7011                   if (dump)
7012                     fprintf (dump, "Emitting stop before label %d\n",
7013                              INSN_UID (last_label));
7014                   emit_insn_before (gen_insn_group_barrier (GEN_INT (3)), last_label);
7015                   insn = last_label;
7016
7017                   init_insn_group_barriers ();
7018                   last_label = 0;
7019                 }
7020             }
7021         }
7022     }
7023 }
7024
7025 /* Like emit_insn_group_barriers, but run if no final scheduling pass was run.
7026    This function has to emit all necessary group barriers.  */
7027
7028 static void
7029 emit_all_insn_group_barriers (FILE *dump ATTRIBUTE_UNUSED)
7030 {
7031   rtx_insn *insn;
7032
7033   init_insn_group_barriers ();
7034
7035   for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
7036     {
7037       if (BARRIER_P (insn))
7038         {
7039           rtx_insn *last = prev_active_insn (insn);
7040
7041           if (! last)
7042             continue;
7043           if (JUMP_TABLE_DATA_P (last))
7044             last = prev_active_insn (last);
7045           if (recog_memoized (last) != CODE_FOR_insn_group_barrier)
7046             emit_insn_after (gen_insn_group_barrier (GEN_INT (3)), last);
7047
7048           init_insn_group_barriers ();
7049         }
7050       else if (NONDEBUG_INSN_P (insn))
7051         {
7052           if (recog_memoized (insn) == CODE_FOR_insn_group_barrier)
7053             init_insn_group_barriers ();
7054           else if (group_barrier_needed (insn))
7055             {
7056               emit_insn_before (gen_insn_group_barrier (GEN_INT (3)), insn);
7057               init_insn_group_barriers ();
7058               group_barrier_needed (insn);
7059             }
7060         }
7061     }
7062 }
7063
7064 \f
7065
7066 /* Instruction scheduling support.  */
7067
7068 #define NR_BUNDLES 10
7069
7070 /* A list of names of all available bundles.  */
7071
7072 static const char *bundle_name [NR_BUNDLES] =
7073 {
7074   ".mii",
7075   ".mmi",
7076   ".mfi",
7077   ".mmf",
7078 #if NR_BUNDLES == 10
7079   ".bbb",
7080   ".mbb",
7081 #endif
7082   ".mib",
7083   ".mmb",
7084   ".mfb",
7085   ".mlx"
7086 };
7087
7088 /* Nonzero if we should insert stop bits into the schedule.  */
7089
7090 int ia64_final_schedule = 0;
7091
7092 /* Codes of the corresponding queried units: */
7093
7094 static int _0mii_, _0mmi_, _0mfi_, _0mmf_;
7095 static int _0bbb_, _0mbb_, _0mib_, _0mmb_, _0mfb_, _0mlx_;
7096
7097 static int _1mii_, _1mmi_, _1mfi_, _1mmf_;
7098 static int _1bbb_, _1mbb_, _1mib_, _1mmb_, _1mfb_, _1mlx_;
7099
7100 static int pos_1, pos_2, pos_3, pos_4, pos_5, pos_6;
7101
7102 /* The following variable value is an insn group barrier.  */
7103
7104 static rtx_insn *dfa_stop_insn;
7105
7106 /* The following variable value is the last issued insn.  */
7107
7108 static rtx_insn *last_scheduled_insn;
7109
7110 /* The following variable value is pointer to a DFA state used as
7111    temporary variable.  */
7112
7113 static state_t temp_dfa_state = NULL;
7114
7115 /* The following variable value is DFA state after issuing the last
7116    insn.  */
7117
7118 static state_t prev_cycle_state = NULL;
7119
7120 /* The following array element values are TRUE if the corresponding
7121    insn requires to add stop bits before it.  */
7122
7123 static char *stops_p = NULL;
7124
7125 /* The following variable is used to set up the mentioned above array.  */
7126
7127 static int stop_before_p = 0;
7128
7129 /* The following variable value is length of the arrays `clocks' and
7130    `add_cycles'. */
7131
7132 static int clocks_length;
7133
7134 /* The following variable value is number of data speculations in progress.  */
7135 static int pending_data_specs = 0;
7136
7137 /* Number of memory references on current and three future processor cycles.  */
7138 static char mem_ops_in_group[4];
7139
7140 /* Number of current processor cycle (from scheduler's point of view).  */
7141 static int current_cycle;
7142
7143 static rtx ia64_single_set (rtx_insn *);
7144 static void ia64_emit_insn_before (rtx, rtx);
7145
7146 /* Map a bundle number to its pseudo-op.  */
7147
7148 const char *
7149 get_bundle_name (int b)
7150 {
7151   return bundle_name[b];
7152 }
7153
7154
7155 /* Return the maximum number of instructions a cpu can issue.  */
7156
7157 static int
7158 ia64_issue_rate (void)
7159 {
7160   return 6;
7161 }
7162
7163 /* Helper function - like single_set, but look inside COND_EXEC.  */
7164
7165 static rtx
7166 ia64_single_set (rtx_insn *insn)
7167 {
7168   rtx x = PATTERN (insn), ret;
7169   if (GET_CODE (x) == COND_EXEC)
7170     x = COND_EXEC_CODE (x);
7171   if (GET_CODE (x) == SET)
7172     return x;
7173
7174   /* Special case here prologue_allocate_stack and epilogue_deallocate_stack.
7175      Although they are not classical single set, the second set is there just
7176      to protect it from moving past FP-relative stack accesses.  */
7177   switch (recog_memoized (insn))
7178     {
7179     case CODE_FOR_prologue_allocate_stack:
7180     case CODE_FOR_prologue_allocate_stack_pr:
7181     case CODE_FOR_epilogue_deallocate_stack:
7182     case CODE_FOR_epilogue_deallocate_stack_pr:
7183       ret = XVECEXP (x, 0, 0);
7184       break;
7185
7186     default:
7187       ret = single_set_2 (insn, x);
7188       break;
7189     }
7190
7191   return ret;
7192 }
7193
7194 /* Adjust the cost of a scheduling dependency.
7195    Return the new cost of a dependency of type DEP_TYPE or INSN on DEP_INSN.
7196    COST is the current cost, DW is dependency weakness.  */
7197 static int
7198 ia64_adjust_cost_2 (rtx_insn *insn, int dep_type1, rtx_insn *dep_insn,
7199                     int cost, dw_t dw)
7200 {
7201   enum reg_note dep_type = (enum reg_note) dep_type1;
7202   enum attr_itanium_class dep_class;
7203   enum attr_itanium_class insn_class;
7204
7205   insn_class = ia64_safe_itanium_class (insn);
7206   dep_class = ia64_safe_itanium_class (dep_insn);
7207
7208   /* Treat true memory dependencies separately.  Ignore apparent true
7209      dependence between store and call (call has a MEM inside a SYMBOL_REF).  */
7210   if (dep_type == REG_DEP_TRUE
7211       && (dep_class == ITANIUM_CLASS_ST || dep_class == ITANIUM_CLASS_STF)
7212       && (insn_class == ITANIUM_CLASS_BR || insn_class == ITANIUM_CLASS_SCALL))
7213     return 0;
7214
7215   if (dw == MIN_DEP_WEAK)
7216     /* Store and load are likely to alias, use higher cost to avoid stall.  */
7217     return PARAM_VALUE (PARAM_SCHED_MEM_TRUE_DEP_COST);
7218   else if (dw > MIN_DEP_WEAK)
7219     {
7220       /* Store and load are less likely to alias.  */
7221       if (mflag_sched_fp_mem_deps_zero_cost && dep_class == ITANIUM_CLASS_STF)
7222         /* Assume there will be no cache conflict for floating-point data.
7223            For integer data, L1 conflict penalty is huge (17 cycles), so we
7224            never assume it will not cause a conflict.  */
7225         return 0;
7226       else
7227         return cost;
7228     }
7229
7230   if (dep_type != REG_DEP_OUTPUT)
7231     return cost;
7232
7233   if (dep_class == ITANIUM_CLASS_ST || dep_class == ITANIUM_CLASS_STF
7234       || insn_class == ITANIUM_CLASS_ST || insn_class == ITANIUM_CLASS_STF)
7235     return 0;
7236
7237   return cost;
7238 }
7239
7240 /* Like emit_insn_before, but skip cycle_display notes.
7241    ??? When cycle display notes are implemented, update this.  */
7242
7243 static void
7244 ia64_emit_insn_before (rtx insn, rtx before)
7245 {
7246   emit_insn_before (insn, before);
7247 }
7248
7249 /* The following function marks insns who produce addresses for load
7250    and store insns.  Such insns will be placed into M slots because it
7251    decrease latency time for Itanium1 (see function
7252    `ia64_produce_address_p' and the DFA descriptions).  */
7253
7254 static void
7255 ia64_dependencies_evaluation_hook (rtx_insn *head, rtx_insn *tail)
7256 {
7257   rtx_insn *insn, *next, *next_tail;
7258
7259   /* Before reload, which_alternative is not set, which means that
7260      ia64_safe_itanium_class will produce wrong results for (at least)
7261      move instructions.  */
7262   if (!reload_completed)
7263     return;
7264
7265   next_tail = NEXT_INSN (tail);
7266   for (insn = head; insn != next_tail; insn = NEXT_INSN (insn))
7267     if (INSN_P (insn))
7268       insn->call = 0;
7269   for (insn = head; insn != next_tail; insn = NEXT_INSN (insn))
7270     if (INSN_P (insn)
7271         && ia64_safe_itanium_class (insn) == ITANIUM_CLASS_IALU)
7272       {
7273         sd_iterator_def sd_it;
7274         dep_t dep;
7275         bool has_mem_op_consumer_p = false;
7276
7277         FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
7278           {
7279             enum attr_itanium_class c;
7280
7281             if (DEP_TYPE (dep) != REG_DEP_TRUE)
7282               continue;
7283
7284             next = DEP_CON (dep);
7285             c = ia64_safe_itanium_class (next);
7286             if ((c == ITANIUM_CLASS_ST
7287                  || c == ITANIUM_CLASS_STF)
7288                 && ia64_st_address_bypass_p (insn, next))
7289               {
7290                 has_mem_op_consumer_p = true;
7291                 break;
7292               }
7293             else if ((c == ITANIUM_CLASS_LD
7294                       || c == ITANIUM_CLASS_FLD
7295                       || c == ITANIUM_CLASS_FLDP)
7296                      && ia64_ld_address_bypass_p (insn, next))
7297               {
7298                 has_mem_op_consumer_p = true;
7299                 break;
7300               }
7301           }
7302
7303         insn->call = has_mem_op_consumer_p;
7304       }
7305 }
7306
7307 /* We're beginning a new block.  Initialize data structures as necessary.  */
7308
7309 static void
7310 ia64_sched_init (FILE *dump ATTRIBUTE_UNUSED,
7311                  int sched_verbose ATTRIBUTE_UNUSED,
7312                  int max_ready ATTRIBUTE_UNUSED)
7313 {
7314 #ifdef ENABLE_CHECKING
7315   rtx_insn *insn;
7316
7317   if (!sel_sched_p () && reload_completed)
7318     for (insn = NEXT_INSN (current_sched_info->prev_head);
7319          insn != current_sched_info->next_tail;
7320          insn = NEXT_INSN (insn))
7321       gcc_assert (!SCHED_GROUP_P (insn));
7322 #endif
7323   last_scheduled_insn = NULL;
7324   init_insn_group_barriers ();
7325
7326   current_cycle = 0;
7327   memset (mem_ops_in_group, 0, sizeof (mem_ops_in_group));
7328 }
7329
7330 /* We're beginning a scheduling pass.  Check assertion.  */
7331
7332 static void
7333 ia64_sched_init_global (FILE *dump ATTRIBUTE_UNUSED,
7334                         int sched_verbose ATTRIBUTE_UNUSED,
7335                         int max_ready ATTRIBUTE_UNUSED)
7336 {
7337   gcc_assert (pending_data_specs == 0);
7338 }
7339
7340 /* Scheduling pass is now finished.  Free/reset static variable.  */
7341 static void
7342 ia64_sched_finish_global (FILE *dump ATTRIBUTE_UNUSED,
7343                           int sched_verbose ATTRIBUTE_UNUSED)
7344 {
7345   gcc_assert (pending_data_specs == 0);
7346 }
7347
7348 /* Return TRUE if INSN is a load (either normal or speculative, but not a
7349    speculation check), FALSE otherwise.  */
7350 static bool
7351 is_load_p (rtx_insn *insn)
7352 {
7353   enum attr_itanium_class insn_class = ia64_safe_itanium_class (insn);
7354
7355   return
7356    ((insn_class == ITANIUM_CLASS_LD || insn_class == ITANIUM_CLASS_FLD)
7357     && get_attr_check_load (insn) == CHECK_LOAD_NO);
7358 }
7359
7360 /* If INSN is a memory reference, memoize it in MEM_OPS_IN_GROUP global array
7361    (taking account for 3-cycle cache reference postponing for stores: Intel
7362    Itanium 2 Reference Manual for Software Development and Optimization,
7363    6.7.3.1).  */
7364 static void
7365 record_memory_reference (rtx_insn *insn)
7366 {
7367   enum attr_itanium_class insn_class = ia64_safe_itanium_class (insn);
7368
7369   switch (insn_class) {
7370     case ITANIUM_CLASS_FLD:
7371     case ITANIUM_CLASS_LD:
7372       mem_ops_in_group[current_cycle % 4]++;
7373       break;
7374     case ITANIUM_CLASS_STF:
7375     case ITANIUM_CLASS_ST:
7376       mem_ops_in_group[(current_cycle + 3) % 4]++;
7377       break;
7378     default:;
7379   }
7380 }
7381
7382 /* We are about to being issuing insns for this clock cycle.
7383    Override the default sort algorithm to better slot instructions.  */
7384
7385 static int
7386 ia64_dfa_sched_reorder (FILE *dump, int sched_verbose, rtx_insn **ready,
7387                         int *pn_ready, int clock_var,
7388                         int reorder_type)
7389 {
7390   int n_asms;
7391   int n_ready = *pn_ready;
7392   rtx_insn **e_ready = ready + n_ready;
7393   rtx_insn **insnp;
7394
7395   if (sched_verbose)
7396     fprintf (dump, "// ia64_dfa_sched_reorder (type %d):\n", reorder_type);
7397
7398   if (reorder_type == 0)
7399     {
7400       /* First, move all USEs, CLOBBERs and other crud out of the way.  */
7401       n_asms = 0;
7402       for (insnp = ready; insnp < e_ready; insnp++)
7403         if (insnp < e_ready)
7404           {
7405             rtx_insn *insn = *insnp;
7406             enum attr_type t = ia64_safe_type (insn);
7407             if (t == TYPE_UNKNOWN)
7408               {
7409                 if (GET_CODE (PATTERN (insn)) == ASM_INPUT
7410                     || asm_noperands (PATTERN (insn)) >= 0)
7411                   {
7412                     rtx_insn *lowest = ready[n_asms];
7413                     ready[n_asms] = insn;
7414                     *insnp = lowest;
7415                     n_asms++;
7416                   }
7417                 else
7418                   {
7419                     rtx_insn *highest = ready[n_ready - 1];
7420                     ready[n_ready - 1] = insn;
7421                     *insnp = highest;
7422                     return 1;
7423                   }
7424               }
7425           }
7426
7427       if (n_asms < n_ready)
7428         {
7429           /* Some normal insns to process.  Skip the asms.  */
7430           ready += n_asms;
7431           n_ready -= n_asms;
7432         }
7433       else if (n_ready > 0)
7434         return 1;
7435     }
7436
7437   if (ia64_final_schedule)
7438     {
7439       int deleted = 0;
7440       int nr_need_stop = 0;
7441
7442       for (insnp = ready; insnp < e_ready; insnp++)
7443         if (safe_group_barrier_needed (*insnp))
7444           nr_need_stop++;
7445
7446       if (reorder_type == 1 && n_ready == nr_need_stop)
7447         return 0;
7448       if (reorder_type == 0)
7449         return 1;
7450       insnp = e_ready;
7451       /* Move down everything that needs a stop bit, preserving
7452          relative order.  */
7453       while (insnp-- > ready + deleted)
7454         while (insnp >= ready + deleted)
7455           {
7456             rtx_insn *insn = *insnp;
7457             if (! safe_group_barrier_needed (insn))
7458               break;
7459             memmove (ready + 1, ready, (insnp - ready) * sizeof (rtx));
7460             *ready = insn;
7461             deleted++;
7462           }
7463       n_ready -= deleted;
7464       ready += deleted;
7465     }
7466
7467   current_cycle = clock_var;
7468   if (reload_completed && mem_ops_in_group[clock_var % 4] >= ia64_max_memory_insns)
7469     {
7470       int moved = 0;
7471
7472       insnp = e_ready;
7473       /* Move down loads/stores, preserving relative order.  */
7474       while (insnp-- > ready + moved)
7475         while (insnp >= ready + moved)
7476           {
7477             rtx_insn *insn = *insnp;
7478             if (! is_load_p (insn))
7479               break;
7480             memmove (ready + 1, ready, (insnp - ready) * sizeof (rtx));
7481             *ready = insn;
7482             moved++;
7483           }
7484       n_ready -= moved;
7485       ready += moved;
7486     }
7487
7488   return 1;
7489 }
7490
7491 /* We are about to being issuing insns for this clock cycle.  Override
7492    the default sort algorithm to better slot instructions.  */
7493
7494 static int
7495 ia64_sched_reorder (FILE *dump, int sched_verbose, rtx_insn **ready,
7496                     int *pn_ready, int clock_var)
7497 {
7498   return ia64_dfa_sched_reorder (dump, sched_verbose, ready,
7499                                  pn_ready, clock_var, 0);
7500 }
7501
7502 /* Like ia64_sched_reorder, but called after issuing each insn.
7503    Override the default sort algorithm to better slot instructions.  */
7504
7505 static int
7506 ia64_sched_reorder2 (FILE *dump ATTRIBUTE_UNUSED,
7507                      int sched_verbose ATTRIBUTE_UNUSED, rtx_insn **ready,
7508                      int *pn_ready, int clock_var)
7509 {
7510   return ia64_dfa_sched_reorder (dump, sched_verbose, ready, pn_ready,
7511                                  clock_var, 1);
7512 }
7513
7514 /* We are about to issue INSN.  Return the number of insns left on the
7515    ready queue that can be issued this cycle.  */
7516
7517 static int
7518 ia64_variable_issue (FILE *dump ATTRIBUTE_UNUSED,
7519                      int sched_verbose ATTRIBUTE_UNUSED,
7520                      rtx_insn *insn,
7521                      int can_issue_more ATTRIBUTE_UNUSED)
7522 {
7523   if (sched_deps_info->generate_spec_deps && !sel_sched_p ())
7524     /* Modulo scheduling does not extend h_i_d when emitting
7525        new instructions.  Don't use h_i_d, if we don't have to.  */
7526     {
7527       if (DONE_SPEC (insn) & BEGIN_DATA)
7528         pending_data_specs++;
7529       if (CHECK_SPEC (insn) & BEGIN_DATA)
7530         pending_data_specs--;
7531     }
7532
7533   if (DEBUG_INSN_P (insn))
7534     return 1;
7535
7536   last_scheduled_insn = insn;
7537   memcpy (prev_cycle_state, curr_state, dfa_state_size);
7538   if (reload_completed)
7539     {
7540       int needed = group_barrier_needed (insn);
7541
7542       gcc_assert (!needed);
7543       if (CALL_P (insn))
7544         init_insn_group_barriers ();
7545       stops_p [INSN_UID (insn)] = stop_before_p;
7546       stop_before_p = 0;
7547
7548       record_memory_reference (insn);
7549     }
7550   return 1;
7551 }
7552
7553 /* We are choosing insn from the ready queue.  Return zero if INSN
7554    can be chosen.  */
7555
7556 static int
7557 ia64_first_cycle_multipass_dfa_lookahead_guard (rtx_insn *insn, int ready_index)
7558 {
7559   gcc_assert (insn && INSN_P (insn));
7560
7561   /* Size of ALAT is 32.  As far as we perform conservative
7562      data speculation, we keep ALAT half-empty.  */
7563   if (pending_data_specs >= 16 && (TODO_SPEC (insn) & BEGIN_DATA))
7564     return ready_index == 0 ? -1 : 1;
7565
7566   if (ready_index == 0)
7567     return 0;
7568
7569   if ((!reload_completed
7570        || !safe_group_barrier_needed (insn))
7571       && (!mflag_sched_mem_insns_hard_limit
7572           || !is_load_p (insn)
7573           || mem_ops_in_group[current_cycle % 4] < ia64_max_memory_insns))
7574     return 0;
7575
7576   return 1;
7577 }
7578
7579 /* The following variable value is pseudo-insn used by the DFA insn
7580    scheduler to change the DFA state when the simulated clock is
7581    increased.  */
7582
7583 static rtx_insn *dfa_pre_cycle_insn;
7584
7585 /* Returns 1 when a meaningful insn was scheduled between the last group
7586    barrier and LAST.  */
7587 static int
7588 scheduled_good_insn (rtx_insn *last)
7589 {
7590   if (last && recog_memoized (last) >= 0)
7591     return 1;
7592
7593   for ( ;
7594        last != NULL && !NOTE_INSN_BASIC_BLOCK_P (last)
7595        && !stops_p[INSN_UID (last)];
7596        last = PREV_INSN (last))
7597     /* We could hit a NOTE_INSN_DELETED here which is actually outside
7598        the ebb we're scheduling.  */
7599     if (INSN_P (last) && recog_memoized (last) >= 0)
7600       return 1;
7601
7602   return 0;
7603 }
7604
7605 /* We are about to being issuing INSN.  Return nonzero if we cannot
7606    issue it on given cycle CLOCK and return zero if we should not sort
7607    the ready queue on the next clock start.  */
7608
7609 static int
7610 ia64_dfa_new_cycle (FILE *dump, int verbose, rtx_insn *insn, int last_clock,
7611                     int clock, int *sort_p)
7612 {
7613   gcc_assert (insn && INSN_P (insn));
7614
7615   if (DEBUG_INSN_P (insn))
7616     return 0;
7617
7618   /* When a group barrier is needed for insn, last_scheduled_insn
7619      should be set.  */
7620   gcc_assert (!(reload_completed && safe_group_barrier_needed (insn))
7621               || last_scheduled_insn);
7622
7623   if ((reload_completed
7624        && (safe_group_barrier_needed (insn)
7625            || (mflag_sched_stop_bits_after_every_cycle
7626                && last_clock != clock
7627                && last_scheduled_insn
7628                && scheduled_good_insn (last_scheduled_insn))))
7629       || (last_scheduled_insn
7630           && (CALL_P (last_scheduled_insn)
7631               || unknown_for_bundling_p (last_scheduled_insn))))
7632     {
7633       init_insn_group_barriers ();
7634
7635       if (verbose && dump)
7636         fprintf (dump, "//    Stop should be before %d%s\n", INSN_UID (insn),
7637                  last_clock == clock ? " + cycle advance" : "");
7638
7639       stop_before_p = 1;
7640       current_cycle = clock;
7641       mem_ops_in_group[current_cycle % 4] = 0;
7642
7643       if (last_clock == clock)
7644         {
7645           state_transition (curr_state, dfa_stop_insn);
7646           if (TARGET_EARLY_STOP_BITS)
7647             *sort_p = (last_scheduled_insn == NULL_RTX
7648                        || ! CALL_P (last_scheduled_insn));
7649           else
7650             *sort_p = 0;
7651           return 1;
7652         }
7653
7654       if (last_scheduled_insn)
7655         {
7656           if (unknown_for_bundling_p (last_scheduled_insn))
7657             state_reset (curr_state);
7658           else
7659             {
7660               memcpy (curr_state, prev_cycle_state, dfa_state_size);
7661               state_transition (curr_state, dfa_stop_insn);
7662               state_transition (curr_state, dfa_pre_cycle_insn);
7663               state_transition (curr_state, NULL);
7664             }
7665         }
7666     }
7667   return 0;
7668 }
7669
7670 /* Implement targetm.sched.h_i_d_extended hook.
7671    Extend internal data structures.  */
7672 static void
7673 ia64_h_i_d_extended (void)
7674 {
7675   if (stops_p != NULL)
7676     {
7677       int new_clocks_length = get_max_uid () * 3 / 2;
7678       stops_p = (char *) xrecalloc (stops_p, new_clocks_length, clocks_length, 1);
7679       clocks_length = new_clocks_length;
7680     }
7681 }
7682 \f
7683
7684 /* This structure describes the data used by the backend to guide scheduling.
7685    When the current scheduling point is switched, this data should be saved
7686    and restored later, if the scheduler returns to this point.  */
7687 struct _ia64_sched_context
7688 {
7689   state_t prev_cycle_state;
7690   rtx_insn *last_scheduled_insn;
7691   struct reg_write_state rws_sum[NUM_REGS];
7692   struct reg_write_state rws_insn[NUM_REGS];
7693   int first_instruction;
7694   int pending_data_specs;
7695   int current_cycle;
7696   char mem_ops_in_group[4];
7697 };
7698 typedef struct _ia64_sched_context *ia64_sched_context_t;
7699
7700 /* Allocates a scheduling context.  */
7701 static void *
7702 ia64_alloc_sched_context (void)
7703 {
7704   return xmalloc (sizeof (struct _ia64_sched_context));
7705 }
7706
7707 /* Initializes the _SC context with clean data, if CLEAN_P, and from
7708    the global context otherwise.  */
7709 static void
7710 ia64_init_sched_context (void *_sc, bool clean_p)
7711 {
7712   ia64_sched_context_t sc = (ia64_sched_context_t) _sc;
7713
7714   sc->prev_cycle_state = xmalloc (dfa_state_size);
7715   if (clean_p)
7716     {
7717       state_reset (sc->prev_cycle_state);
7718       sc->last_scheduled_insn = NULL;
7719       memset (sc->rws_sum, 0, sizeof (rws_sum));
7720       memset (sc->rws_insn, 0, sizeof (rws_insn));
7721       sc->first_instruction = 1;
7722       sc->pending_data_specs = 0;
7723       sc->current_cycle = 0;
7724       memset (sc->mem_ops_in_group, 0, sizeof (mem_ops_in_group));
7725     }
7726   else
7727     {
7728       memcpy (sc->prev_cycle_state, prev_cycle_state, dfa_state_size);
7729       sc->last_scheduled_insn = last_scheduled_insn;
7730       memcpy (sc->rws_sum, rws_sum, sizeof (rws_sum));
7731       memcpy (sc->rws_insn, rws_insn, sizeof (rws_insn));
7732       sc->first_instruction = first_instruction;
7733       sc->pending_data_specs = pending_data_specs;
7734       sc->current_cycle = current_cycle;
7735       memcpy (sc->mem_ops_in_group, mem_ops_in_group, sizeof (mem_ops_in_group));
7736     }
7737 }
7738
7739 /* Sets the global scheduling context to the one pointed to by _SC.  */
7740 static void
7741 ia64_set_sched_context (void *_sc)
7742 {
7743   ia64_sched_context_t sc = (ia64_sched_context_t) _sc;
7744
7745   gcc_assert (sc != NULL);
7746
7747   memcpy (prev_cycle_state, sc->prev_cycle_state, dfa_state_size);
7748   last_scheduled_insn = sc->last_scheduled_insn;
7749   memcpy (rws_sum, sc->rws_sum, sizeof (rws_sum));
7750   memcpy (rws_insn, sc->rws_insn, sizeof (rws_insn));
7751   first_instruction = sc->first_instruction;
7752   pending_data_specs = sc->pending_data_specs;
7753   current_cycle = sc->current_cycle;
7754   memcpy (mem_ops_in_group, sc->mem_ops_in_group, sizeof (mem_ops_in_group));
7755 }
7756
7757 /* Clears the data in the _SC scheduling context.  */
7758 static void
7759 ia64_clear_sched_context (void *_sc)
7760 {
7761   ia64_sched_context_t sc = (ia64_sched_context_t) _sc;
7762
7763   free (sc->prev_cycle_state);
7764   sc->prev_cycle_state = NULL;
7765 }
7766
7767 /* Frees the _SC scheduling context.  */
7768 static void
7769 ia64_free_sched_context (void *_sc)
7770 {
7771   gcc_assert (_sc != NULL);
7772
7773   free (_sc);
7774 }
7775
7776 typedef rtx (* gen_func_t) (rtx, rtx);
7777
7778 /* Return a function that will generate a load of mode MODE_NO
7779    with speculation types TS.  */
7780 static gen_func_t
7781 get_spec_load_gen_function (ds_t ts, int mode_no)
7782 {
7783   static gen_func_t gen_ld_[] = {
7784     gen_movbi,
7785     gen_movqi_internal,
7786     gen_movhi_internal,
7787     gen_movsi_internal,
7788     gen_movdi_internal,
7789     gen_movsf_internal,
7790     gen_movdf_internal,
7791     gen_movxf_internal,
7792     gen_movti_internal,
7793     gen_zero_extendqidi2,
7794     gen_zero_extendhidi2,
7795     gen_zero_extendsidi2,
7796   };
7797
7798   static gen_func_t gen_ld_a[] = {
7799     gen_movbi_advanced,
7800     gen_movqi_advanced,
7801     gen_movhi_advanced,
7802     gen_movsi_advanced,
7803     gen_movdi_advanced,
7804     gen_movsf_advanced,
7805     gen_movdf_advanced,
7806     gen_movxf_advanced,
7807     gen_movti_advanced,
7808     gen_zero_extendqidi2_advanced,
7809     gen_zero_extendhidi2_advanced,
7810     gen_zero_extendsidi2_advanced,
7811   };
7812   static gen_func_t gen_ld_s[] = {
7813     gen_movbi_speculative,
7814     gen_movqi_speculative,
7815     gen_movhi_speculative,
7816     gen_movsi_speculative,
7817     gen_movdi_speculative,
7818     gen_movsf_speculative,
7819     gen_movdf_speculative,
7820     gen_movxf_speculative,
7821     gen_movti_speculative,
7822     gen_zero_extendqidi2_speculative,
7823     gen_zero_extendhidi2_speculative,
7824     gen_zero_extendsidi2_speculative,
7825   };
7826   static gen_func_t gen_ld_sa[] = {
7827     gen_movbi_speculative_advanced,
7828     gen_movqi_speculative_advanced,
7829     gen_movhi_speculative_advanced,
7830     gen_movsi_speculative_advanced,
7831     gen_movdi_speculative_advanced,
7832     gen_movsf_speculative_advanced,
7833     gen_movdf_speculative_advanced,
7834     gen_movxf_speculative_advanced,
7835     gen_movti_speculative_advanced,
7836     gen_zero_extendqidi2_speculative_advanced,
7837     gen_zero_extendhidi2_speculative_advanced,
7838     gen_zero_extendsidi2_speculative_advanced,
7839   };
7840   static gen_func_t gen_ld_s_a[] = {
7841     gen_movbi_speculative_a,
7842     gen_movqi_speculative_a,
7843     gen_movhi_speculative_a,
7844     gen_movsi_speculative_a,
7845     gen_movdi_speculative_a,
7846     gen_movsf_speculative_a,
7847     gen_movdf_speculative_a,
7848     gen_movxf_speculative_a,
7849     gen_movti_speculative_a,
7850     gen_zero_extendqidi2_speculative_a,
7851     gen_zero_extendhidi2_speculative_a,
7852     gen_zero_extendsidi2_speculative_a,
7853   };
7854
7855   gen_func_t *gen_ld;
7856
7857   if (ts & BEGIN_DATA)
7858     {
7859       if (ts & BEGIN_CONTROL)
7860         gen_ld = gen_ld_sa;
7861       else
7862         gen_ld = gen_ld_a;
7863     }
7864   else if (ts & BEGIN_CONTROL)
7865     {
7866       if ((spec_info->flags & SEL_SCHED_SPEC_DONT_CHECK_CONTROL)
7867           || ia64_needs_block_p (ts))
7868         gen_ld = gen_ld_s;
7869       else
7870         gen_ld = gen_ld_s_a;
7871     }
7872   else if (ts == 0)
7873     gen_ld = gen_ld_;
7874   else
7875     gcc_unreachable ();
7876
7877   return gen_ld[mode_no];
7878 }
7879
7880 /* Constants that help mapping 'machine_mode' to int.  */
7881 enum SPEC_MODES
7882   {
7883     SPEC_MODE_INVALID = -1,
7884     SPEC_MODE_FIRST = 0,
7885     SPEC_MODE_FOR_EXTEND_FIRST = 1,
7886     SPEC_MODE_FOR_EXTEND_LAST = 3,
7887     SPEC_MODE_LAST = 8
7888   };
7889
7890 enum
7891   {
7892     /* Offset to reach ZERO_EXTEND patterns.  */
7893     SPEC_GEN_EXTEND_OFFSET = SPEC_MODE_LAST - SPEC_MODE_FOR_EXTEND_FIRST + 1
7894   };
7895
7896 /* Return index of the MODE.  */
7897 static int
7898 ia64_mode_to_int (machine_mode mode)
7899 {
7900   switch (mode)
7901     {
7902     case BImode: return 0; /* SPEC_MODE_FIRST  */
7903     case QImode: return 1; /* SPEC_MODE_FOR_EXTEND_FIRST  */
7904     case HImode: return 2;
7905     case SImode: return 3; /* SPEC_MODE_FOR_EXTEND_LAST  */
7906     case DImode: return 4;
7907     case SFmode: return 5;
7908     case DFmode: return 6;
7909     case XFmode: return 7;
7910     case TImode:
7911       /* ??? This mode needs testing.  Bypasses for ldfp8 instruction are not
7912          mentioned in itanium[12].md.  Predicate fp_register_operand also
7913          needs to be defined.  Bottom line: better disable for now.  */
7914       return SPEC_MODE_INVALID;
7915     default:     return SPEC_MODE_INVALID;
7916     }
7917 }
7918
7919 /* Provide information about speculation capabilities.  */
7920 static void
7921 ia64_set_sched_flags (spec_info_t spec_info)
7922 {
7923   unsigned int *flags = &(current_sched_info->flags);
7924
7925   if (*flags & SCHED_RGN
7926       || *flags & SCHED_EBB
7927       || *flags & SEL_SCHED)
7928     {
7929       int mask = 0;
7930
7931       if ((mflag_sched_br_data_spec && !reload_completed && optimize > 0)
7932           || (mflag_sched_ar_data_spec && reload_completed))
7933         {
7934           mask |= BEGIN_DATA;
7935
7936           if (!sel_sched_p ()
7937               && ((mflag_sched_br_in_data_spec && !reload_completed)
7938                   || (mflag_sched_ar_in_data_spec && reload_completed)))
7939             mask |= BE_IN_DATA;
7940         }
7941
7942       if (mflag_sched_control_spec
7943           && (!sel_sched_p ()
7944               || reload_completed))
7945         {
7946           mask |= BEGIN_CONTROL;
7947
7948           if (!sel_sched_p () && mflag_sched_in_control_spec)
7949             mask |= BE_IN_CONTROL;
7950         }
7951
7952       spec_info->mask = mask;
7953
7954       if (mask)
7955         {
7956           *flags |= USE_DEPS_LIST | DO_SPECULATION;
7957
7958           if (mask & BE_IN_SPEC)
7959             *flags |= NEW_BBS;
7960
7961           spec_info->flags = 0;
7962
7963           if ((mask & CONTROL_SPEC)
7964               && sel_sched_p () && mflag_sel_sched_dont_check_control_spec)
7965             spec_info->flags |= SEL_SCHED_SPEC_DONT_CHECK_CONTROL;
7966
7967           if (sched_verbose >= 1)
7968             spec_info->dump = sched_dump;
7969           else
7970             spec_info->dump = 0;
7971
7972           if (mflag_sched_count_spec_in_critical_path)
7973             spec_info->flags |= COUNT_SPEC_IN_CRITICAL_PATH;
7974         }
7975     }
7976   else
7977     spec_info->mask = 0;
7978 }
7979
7980 /* If INSN is an appropriate load return its mode.
7981    Return -1 otherwise.  */
7982 static int
7983 get_mode_no_for_insn (rtx_insn *insn)
7984 {
7985   rtx reg, mem, mode_rtx;
7986   int mode_no;
7987   bool extend_p;
7988
7989   extract_insn_cached (insn);
7990
7991   /* We use WHICH_ALTERNATIVE only after reload.  This will
7992      guarantee that reload won't touch a speculative insn.  */
7993
7994   if (recog_data.n_operands != 2)
7995     return -1;
7996
7997   reg = recog_data.operand[0];
7998   mem = recog_data.operand[1];
7999
8000   /* We should use MEM's mode since REG's mode in presence of
8001      ZERO_EXTEND will always be DImode.  */
8002   if (get_attr_speculable1 (insn) == SPECULABLE1_YES)
8003     /* Process non-speculative ld.  */
8004     {
8005       if (!reload_completed)
8006         {
8007           /* Do not speculate into regs like ar.lc.  */
8008           if (!REG_P (reg) || AR_REGNO_P (REGNO (reg)))
8009             return -1;
8010
8011           if (!MEM_P (mem))
8012             return -1;
8013
8014           {
8015             rtx mem_reg = XEXP (mem, 0);
8016
8017             if (!REG_P (mem_reg))
8018               return -1;
8019           }
8020
8021           mode_rtx = mem;
8022         }
8023       else if (get_attr_speculable2 (insn) == SPECULABLE2_YES)
8024         {
8025           gcc_assert (REG_P (reg) && MEM_P (mem));
8026           mode_rtx = mem;
8027         }
8028       else
8029         return -1;
8030     }
8031   else if (get_attr_data_speculative (insn) == DATA_SPECULATIVE_YES
8032            || get_attr_control_speculative (insn) == CONTROL_SPECULATIVE_YES
8033            || get_attr_check_load (insn) == CHECK_LOAD_YES)
8034     /* Process speculative ld or ld.c.  */
8035     {
8036       gcc_assert (REG_P (reg) && MEM_P (mem));
8037       mode_rtx = mem;
8038     }
8039   else
8040     {
8041       enum attr_itanium_class attr_class = get_attr_itanium_class (insn);
8042
8043       if (attr_class == ITANIUM_CLASS_CHK_A
8044           || attr_class == ITANIUM_CLASS_CHK_S_I
8045           || attr_class == ITANIUM_CLASS_CHK_S_F)
8046         /* Process chk.  */
8047         mode_rtx = reg;
8048       else
8049         return -1;
8050     }
8051
8052   mode_no = ia64_mode_to_int (GET_MODE (mode_rtx));
8053
8054   if (mode_no == SPEC_MODE_INVALID)
8055     return -1;
8056
8057   extend_p = (GET_MODE (reg) != GET_MODE (mode_rtx));
8058
8059   if (extend_p)
8060     {
8061       if (!(SPEC_MODE_FOR_EXTEND_FIRST <= mode_no
8062             && mode_no <= SPEC_MODE_FOR_EXTEND_LAST))
8063         return -1;
8064
8065       mode_no += SPEC_GEN_EXTEND_OFFSET;
8066     }
8067
8068   return mode_no;
8069 }
8070
8071 /* If X is an unspec part of a speculative load, return its code.
8072    Return -1 otherwise.  */
8073 static int
8074 get_spec_unspec_code (const_rtx x)
8075 {
8076   if (GET_CODE (x) != UNSPEC)
8077     return -1;
8078
8079   {
8080     int code;
8081
8082     code = XINT (x, 1);
8083
8084     switch (code)
8085       {
8086       case UNSPEC_LDA:
8087       case UNSPEC_LDS:
8088       case UNSPEC_LDS_A:
8089       case UNSPEC_LDSA:
8090         return code;
8091
8092       default:
8093         return -1;
8094       }
8095   }
8096 }
8097
8098 /* Implement skip_rtx_p hook.  */
8099 static bool
8100 ia64_skip_rtx_p (const_rtx x)
8101 {
8102   return get_spec_unspec_code (x) != -1;
8103 }
8104
8105 /* If INSN is a speculative load, return its UNSPEC code.
8106    Return -1 otherwise.  */
8107 static int
8108 get_insn_spec_code (const_rtx insn)
8109 {
8110   rtx pat, reg, mem;
8111
8112   pat = PATTERN (insn);
8113
8114   if (GET_CODE (pat) == COND_EXEC)
8115     pat = COND_EXEC_CODE (pat);
8116
8117   if (GET_CODE (pat) != SET)
8118     return -1;
8119
8120   reg = SET_DEST (pat);
8121   if (!REG_P (reg))
8122     return -1;
8123
8124   mem = SET_SRC (pat);
8125   if (GET_CODE (mem) == ZERO_EXTEND)
8126     mem = XEXP (mem, 0);
8127
8128   return get_spec_unspec_code (mem);
8129 }
8130
8131 /* If INSN is a speculative load, return a ds with the speculation types.
8132    Otherwise [if INSN is a normal instruction] return 0.  */
8133 static ds_t
8134 ia64_get_insn_spec_ds (rtx_insn *insn)
8135 {
8136   int code = get_insn_spec_code (insn);
8137
8138   switch (code)
8139     {
8140     case UNSPEC_LDA:
8141       return BEGIN_DATA;
8142
8143     case UNSPEC_LDS:
8144     case UNSPEC_LDS_A:
8145       return BEGIN_CONTROL;
8146
8147     case UNSPEC_LDSA:
8148       return BEGIN_DATA | BEGIN_CONTROL;
8149
8150     default:
8151       return 0;
8152     }
8153 }
8154
8155 /* If INSN is a speculative load return a ds with the speculation types that
8156    will be checked.
8157    Otherwise [if INSN is a normal instruction] return 0.  */
8158 static ds_t
8159 ia64_get_insn_checked_ds (rtx_insn *insn)
8160 {
8161   int code = get_insn_spec_code (insn);
8162
8163   switch (code)
8164     {
8165     case UNSPEC_LDA:
8166       return BEGIN_DATA | BEGIN_CONTROL;
8167
8168     case UNSPEC_LDS:
8169       return BEGIN_CONTROL;
8170
8171     case UNSPEC_LDS_A:
8172     case UNSPEC_LDSA:
8173       return BEGIN_DATA | BEGIN_CONTROL;
8174
8175     default:
8176       return 0;
8177     }
8178 }
8179
8180 /* If GEN_P is true, calculate the index of needed speculation check and return
8181    speculative pattern for INSN with speculative mode TS, machine mode
8182    MODE_NO and with ZERO_EXTEND (if EXTEND_P is true).
8183    If GEN_P is false, just calculate the index of needed speculation check.  */
8184 static rtx
8185 ia64_gen_spec_load (rtx insn, ds_t ts, int mode_no)
8186 {
8187   rtx pat, new_pat;
8188   gen_func_t gen_load;
8189
8190   gen_load = get_spec_load_gen_function (ts, mode_no);
8191
8192   new_pat = gen_load (copy_rtx (recog_data.operand[0]),
8193                       copy_rtx (recog_data.operand[1]));
8194
8195   pat = PATTERN (insn);
8196   if (GET_CODE (pat) == COND_EXEC)
8197     new_pat = gen_rtx_COND_EXEC (VOIDmode, copy_rtx (COND_EXEC_TEST (pat)),
8198                                  new_pat);
8199
8200   return new_pat;
8201 }
8202
8203 static bool
8204 insn_can_be_in_speculative_p (rtx insn ATTRIBUTE_UNUSED,
8205                               ds_t ds ATTRIBUTE_UNUSED)
8206 {
8207   return false;
8208 }
8209
8210 /* Implement targetm.sched.speculate_insn hook.
8211    Check if the INSN can be TS speculative.
8212    If 'no' - return -1.
8213    If 'yes' - generate speculative pattern in the NEW_PAT and return 1.
8214    If current pattern of the INSN already provides TS speculation,
8215    return 0.  */
8216 static int
8217 ia64_speculate_insn (rtx_insn *insn, ds_t ts, rtx *new_pat)
8218 {
8219   int mode_no;
8220   int res;
8221
8222   gcc_assert (!(ts & ~SPECULATIVE));
8223
8224   if (ia64_spec_check_p (insn))
8225     return -1;
8226
8227   if ((ts & BE_IN_SPEC)
8228       && !insn_can_be_in_speculative_p (insn, ts))
8229     return -1;
8230
8231   mode_no = get_mode_no_for_insn (insn);
8232
8233   if (mode_no != SPEC_MODE_INVALID)
8234     {
8235       if (ia64_get_insn_spec_ds (insn) == ds_get_speculation_types (ts))
8236         res = 0;
8237       else
8238         {
8239           res = 1;
8240           *new_pat = ia64_gen_spec_load (insn, ts, mode_no);
8241         }
8242     }
8243   else
8244     res = -1;
8245
8246   return res;
8247 }
8248
8249 /* Return a function that will generate a check for speculation TS with mode
8250    MODE_NO.
8251    If simple check is needed, pass true for SIMPLE_CHECK_P.
8252    If clearing check is needed, pass true for CLEARING_CHECK_P.  */
8253 static gen_func_t
8254 get_spec_check_gen_function (ds_t ts, int mode_no,
8255                              bool simple_check_p, bool clearing_check_p)
8256 {
8257   static gen_func_t gen_ld_c_clr[] = {
8258     gen_movbi_clr,
8259     gen_movqi_clr,
8260     gen_movhi_clr,
8261     gen_movsi_clr,
8262     gen_movdi_clr,
8263     gen_movsf_clr,
8264     gen_movdf_clr,
8265     gen_movxf_clr,
8266     gen_movti_clr,
8267     gen_zero_extendqidi2_clr,
8268     gen_zero_extendhidi2_clr,
8269     gen_zero_extendsidi2_clr,
8270   };
8271   static gen_func_t gen_ld_c_nc[] = {
8272     gen_movbi_nc,
8273     gen_movqi_nc,
8274     gen_movhi_nc,
8275     gen_movsi_nc,
8276     gen_movdi_nc,
8277     gen_movsf_nc,
8278     gen_movdf_nc,
8279     gen_movxf_nc,
8280     gen_movti_nc,
8281     gen_zero_extendqidi2_nc,
8282     gen_zero_extendhidi2_nc,
8283     gen_zero_extendsidi2_nc,
8284   };
8285   static gen_func_t gen_chk_a_clr[] = {
8286     gen_advanced_load_check_clr_bi,
8287     gen_advanced_load_check_clr_qi,
8288     gen_advanced_load_check_clr_hi,
8289     gen_advanced_load_check_clr_si,
8290     gen_advanced_load_check_clr_di,
8291     gen_advanced_load_check_clr_sf,
8292     gen_advanced_load_check_clr_df,
8293     gen_advanced_load_check_clr_xf,
8294     gen_advanced_load_check_clr_ti,
8295     gen_advanced_load_check_clr_di,
8296     gen_advanced_load_check_clr_di,
8297     gen_advanced_load_check_clr_di,
8298   };
8299   static gen_func_t gen_chk_a_nc[] = {
8300     gen_advanced_load_check_nc_bi,
8301     gen_advanced_load_check_nc_qi,
8302     gen_advanced_load_check_nc_hi,
8303     gen_advanced_load_check_nc_si,
8304     gen_advanced_load_check_nc_di,
8305     gen_advanced_load_check_nc_sf,
8306     gen_advanced_load_check_nc_df,
8307     gen_advanced_load_check_nc_xf,
8308     gen_advanced_load_check_nc_ti,
8309     gen_advanced_load_check_nc_di,
8310     gen_advanced_load_check_nc_di,
8311     gen_advanced_load_check_nc_di,
8312   };
8313   static gen_func_t gen_chk_s[] = {
8314     gen_speculation_check_bi,
8315     gen_speculation_check_qi,
8316     gen_speculation_check_hi,
8317     gen_speculation_check_si,
8318     gen_speculation_check_di,
8319     gen_speculation_check_sf,
8320     gen_speculation_check_df,
8321     gen_speculation_check_xf,
8322     gen_speculation_check_ti,
8323     gen_speculation_check_di,
8324     gen_speculation_check_di,
8325     gen_speculation_check_di,
8326   };
8327
8328   gen_func_t *gen_check;
8329
8330   if (ts & BEGIN_DATA)
8331     {
8332       /* We don't need recovery because even if this is ld.sa
8333          ALAT entry will be allocated only if NAT bit is set to zero.
8334          So it is enough to use ld.c here.  */
8335
8336       if (simple_check_p)
8337         {
8338           gcc_assert (mflag_sched_spec_ldc);
8339
8340           if (clearing_check_p)
8341             gen_check = gen_ld_c_clr;
8342           else
8343             gen_check = gen_ld_c_nc;
8344         }
8345       else
8346         {
8347           if (clearing_check_p)
8348             gen_check = gen_chk_a_clr;
8349           else
8350             gen_check = gen_chk_a_nc;
8351         }
8352     }
8353   else if (ts & BEGIN_CONTROL)
8354     {
8355       if (simple_check_p)
8356         /* We might want to use ld.sa -> ld.c instead of
8357            ld.s -> chk.s.  */
8358         {
8359           gcc_assert (!ia64_needs_block_p (ts));
8360
8361           if (clearing_check_p)
8362             gen_check = gen_ld_c_clr;
8363           else
8364             gen_check = gen_ld_c_nc;
8365         }
8366       else
8367         {
8368           gen_check = gen_chk_s;
8369         }
8370     }
8371   else
8372     gcc_unreachable ();
8373
8374   gcc_assert (mode_no >= 0);
8375   return gen_check[mode_no];
8376 }
8377
8378 /* Return nonzero, if INSN needs branchy recovery check.  */
8379 static bool
8380 ia64_needs_block_p (ds_t ts)
8381 {
8382   if (ts & BEGIN_DATA)
8383     return !mflag_sched_spec_ldc;
8384
8385   gcc_assert ((ts & BEGIN_CONTROL) != 0);
8386
8387   return !(mflag_sched_spec_control_ldc && mflag_sched_spec_ldc);
8388 }
8389
8390 /* Generate (or regenerate) a recovery check for INSN.  */
8391 static rtx
8392 ia64_gen_spec_check (rtx_insn *insn, rtx_insn *label, ds_t ds)
8393 {
8394   rtx op1, pat, check_pat;
8395   gen_func_t gen_check;
8396   int mode_no;
8397
8398   mode_no = get_mode_no_for_insn (insn);
8399   gcc_assert (mode_no >= 0);
8400
8401   if (label)
8402     op1 = label;
8403   else
8404     {
8405       gcc_assert (!ia64_needs_block_p (ds));
8406       op1 = copy_rtx (recog_data.operand[1]);
8407     }
8408
8409   gen_check = get_spec_check_gen_function (ds, mode_no, label == NULL_RTX,
8410                                            true);
8411
8412   check_pat = gen_check (copy_rtx (recog_data.operand[0]), op1);
8413
8414   pat = PATTERN (insn);
8415   if (GET_CODE (pat) == COND_EXEC)
8416     check_pat = gen_rtx_COND_EXEC (VOIDmode, copy_rtx (COND_EXEC_TEST (pat)),
8417                                    check_pat);
8418
8419   return check_pat;
8420 }
8421
8422 /* Return nonzero, if X is branchy recovery check.  */
8423 static int
8424 ia64_spec_check_p (rtx x)
8425 {
8426   x = PATTERN (x);
8427   if (GET_CODE (x) == COND_EXEC)
8428     x = COND_EXEC_CODE (x);
8429   if (GET_CODE (x) == SET)
8430     return ia64_spec_check_src_p (SET_SRC (x));
8431   return 0;
8432 }
8433
8434 /* Return nonzero, if SRC belongs to recovery check.  */
8435 static int
8436 ia64_spec_check_src_p (rtx src)
8437 {
8438   if (GET_CODE (src) == IF_THEN_ELSE)
8439     {
8440       rtx t;
8441
8442       t = XEXP (src, 0);
8443       if (GET_CODE (t) == NE)
8444         {
8445           t = XEXP (t, 0);
8446
8447           if (GET_CODE (t) == UNSPEC)
8448             {
8449               int code;
8450
8451               code = XINT (t, 1);
8452
8453               if (code == UNSPEC_LDCCLR
8454                   || code == UNSPEC_LDCNC
8455                   || code == UNSPEC_CHKACLR
8456                   || code == UNSPEC_CHKANC
8457                   || code == UNSPEC_CHKS)
8458                 {
8459                   gcc_assert (code != 0);
8460                   return code;
8461                 }
8462             }
8463         }
8464     }
8465   return 0;
8466 }
8467 \f
8468
8469 /* The following page contains abstract data `bundle states' which are
8470    used for bundling insns (inserting nops and template generation).  */
8471
8472 /* The following describes state of insn bundling.  */
8473
8474 struct bundle_state
8475 {
8476   /* Unique bundle state number to identify them in the debugging
8477      output  */
8478   int unique_num;
8479   rtx_insn *insn; /* corresponding insn, NULL for the 1st and the last state  */
8480   /* number nops before and after the insn  */
8481   short before_nops_num, after_nops_num;
8482   int insn_num; /* insn number (0 - for initial state, 1 - for the 1st
8483                    insn */
8484   int cost;     /* cost of the state in cycles */
8485   int accumulated_insns_num; /* number of all previous insns including
8486                                 nops.  L is considered as 2 insns */
8487   int branch_deviation; /* deviation of previous branches from 3rd slots  */
8488   int middle_bundle_stops; /* number of stop bits in the middle of bundles */
8489   struct bundle_state *next;  /* next state with the same insn_num  */
8490   struct bundle_state *originator; /* originator (previous insn state)  */
8491   /* All bundle states are in the following chain.  */
8492   struct bundle_state *allocated_states_chain;
8493   /* The DFA State after issuing the insn and the nops.  */
8494   state_t dfa_state;
8495 };
8496
8497 /* The following is map insn number to the corresponding bundle state.  */
8498
8499 static struct bundle_state **index_to_bundle_states;
8500
8501 /* The unique number of next bundle state.  */
8502
8503 static int bundle_states_num;
8504
8505 /* All allocated bundle states are in the following chain.  */
8506
8507 static struct bundle_state *allocated_bundle_states_chain;
8508
8509 /* All allocated but not used bundle states are in the following
8510    chain.  */
8511
8512 static struct bundle_state *free_bundle_state_chain;
8513
8514
8515 /* The following function returns a free bundle state.  */
8516
8517 static struct bundle_state *
8518 get_free_bundle_state (void)
8519 {
8520   struct bundle_state *result;
8521
8522   if (free_bundle_state_chain != NULL)
8523     {
8524       result = free_bundle_state_chain;
8525       free_bundle_state_chain = result->next;
8526     }
8527   else
8528     {
8529       result = XNEW (struct bundle_state);
8530       result->dfa_state = xmalloc (dfa_state_size);
8531       result->allocated_states_chain = allocated_bundle_states_chain;
8532       allocated_bundle_states_chain = result;
8533     }
8534   result->unique_num = bundle_states_num++;
8535   return result;
8536
8537 }
8538
8539 /* The following function frees given bundle state.  */
8540
8541 static void
8542 free_bundle_state (struct bundle_state *state)
8543 {
8544   state->next = free_bundle_state_chain;
8545   free_bundle_state_chain = state;
8546 }
8547
8548 /* Start work with abstract data `bundle states'.  */
8549
8550 static void
8551 initiate_bundle_states (void)
8552 {
8553   bundle_states_num = 0;
8554   free_bundle_state_chain = NULL;
8555   allocated_bundle_states_chain = NULL;
8556 }
8557
8558 /* Finish work with abstract data `bundle states'.  */
8559
8560 static void
8561 finish_bundle_states (void)
8562 {
8563   struct bundle_state *curr_state, *next_state;
8564
8565   for (curr_state = allocated_bundle_states_chain;
8566        curr_state != NULL;
8567        curr_state = next_state)
8568     {
8569       next_state = curr_state->allocated_states_chain;
8570       free (curr_state->dfa_state);
8571       free (curr_state);
8572     }
8573 }
8574
8575 /* Hashtable helpers.  */
8576
8577 struct bundle_state_hasher : typed_noop_remove <bundle_state>
8578 {
8579   typedef bundle_state value_type;
8580   typedef bundle_state compare_type;
8581   static inline hashval_t hash (const value_type *);
8582   static inline bool equal (const value_type *, const compare_type *);
8583 };
8584
8585 /* The function returns hash of BUNDLE_STATE.  */
8586
8587 inline hashval_t
8588 bundle_state_hasher::hash (const value_type *state)
8589 {
8590   unsigned result, i;
8591
8592   for (result = i = 0; i < dfa_state_size; i++)
8593     result += (((unsigned char *) state->dfa_state) [i]
8594                << ((i % CHAR_BIT) * 3 + CHAR_BIT));
8595   return result + state->insn_num;
8596 }
8597
8598 /* The function returns nonzero if the bundle state keys are equal.  */
8599
8600 inline bool
8601 bundle_state_hasher::equal (const value_type *state1,
8602                             const compare_type *state2)
8603 {
8604   return (state1->insn_num == state2->insn_num
8605           && memcmp (state1->dfa_state, state2->dfa_state,
8606                      dfa_state_size) == 0);
8607 }
8608
8609 /* Hash table of the bundle states.  The key is dfa_state and insn_num
8610    of the bundle states.  */
8611
8612 static hash_table<bundle_state_hasher> *bundle_state_table;
8613
8614 /* The function inserts the BUNDLE_STATE into the hash table.  The
8615    function returns nonzero if the bundle has been inserted into the
8616    table.  The table contains the best bundle state with given key.  */
8617
8618 static int
8619 insert_bundle_state (struct bundle_state *bundle_state)
8620 {
8621   struct bundle_state **entry_ptr;
8622
8623   entry_ptr = bundle_state_table->find_slot (bundle_state, INSERT);
8624   if (*entry_ptr == NULL)
8625     {
8626       bundle_state->next = index_to_bundle_states [bundle_state->insn_num];
8627       index_to_bundle_states [bundle_state->insn_num] = bundle_state;
8628       *entry_ptr = bundle_state;
8629       return TRUE;
8630     }
8631   else if (bundle_state->cost < (*entry_ptr)->cost
8632            || (bundle_state->cost == (*entry_ptr)->cost
8633                && ((*entry_ptr)->accumulated_insns_num
8634                    > bundle_state->accumulated_insns_num
8635                    || ((*entry_ptr)->accumulated_insns_num
8636                        == bundle_state->accumulated_insns_num
8637                        && ((*entry_ptr)->branch_deviation
8638                            > bundle_state->branch_deviation
8639                            || ((*entry_ptr)->branch_deviation
8640                                == bundle_state->branch_deviation
8641                                && (*entry_ptr)->middle_bundle_stops
8642                                > bundle_state->middle_bundle_stops))))))
8643
8644     {
8645       struct bundle_state temp;
8646
8647       temp = **entry_ptr;
8648       **entry_ptr = *bundle_state;
8649       (*entry_ptr)->next = temp.next;
8650       *bundle_state = temp;
8651     }
8652   return FALSE;
8653 }
8654
8655 /* Start work with the hash table.  */
8656
8657 static void
8658 initiate_bundle_state_table (void)
8659 {
8660   bundle_state_table = new hash_table<bundle_state_hasher> (50);
8661 }
8662
8663 /* Finish work with the hash table.  */
8664
8665 static void
8666 finish_bundle_state_table (void)
8667 {
8668   delete bundle_state_table;
8669   bundle_state_table = NULL;
8670 }
8671
8672 \f
8673
8674 /* The following variable is a insn `nop' used to check bundle states
8675    with different number of inserted nops.  */
8676
8677 static rtx_insn *ia64_nop;
8678
8679 /* The following function tries to issue NOPS_NUM nops for the current
8680    state without advancing processor cycle.  If it failed, the
8681    function returns FALSE and frees the current state.  */
8682
8683 static int
8684 try_issue_nops (struct bundle_state *curr_state, int nops_num)
8685 {
8686   int i;
8687
8688   for (i = 0; i < nops_num; i++)
8689     if (state_transition (curr_state->dfa_state, ia64_nop) >= 0)
8690       {
8691         free_bundle_state (curr_state);
8692         return FALSE;
8693       }
8694   return TRUE;
8695 }
8696
8697 /* The following function tries to issue INSN for the current
8698    state without advancing processor cycle.  If it failed, the
8699    function returns FALSE and frees the current state.  */
8700
8701 static int
8702 try_issue_insn (struct bundle_state *curr_state, rtx insn)
8703 {
8704   if (insn && state_transition (curr_state->dfa_state, insn) >= 0)
8705     {
8706       free_bundle_state (curr_state);
8707       return FALSE;
8708     }
8709   return TRUE;
8710 }
8711
8712 /* The following function tries to issue BEFORE_NOPS_NUM nops and INSN
8713    starting with ORIGINATOR without advancing processor cycle.  If
8714    TRY_BUNDLE_END_P is TRUE, the function also/only (if
8715    ONLY_BUNDLE_END_P is TRUE) tries to issue nops to fill all bundle.
8716    If it was successful, the function creates new bundle state and
8717    insert into the hash table and into `index_to_bundle_states'.  */
8718
8719 static void
8720 issue_nops_and_insn (struct bundle_state *originator, int before_nops_num,
8721                      rtx_insn *insn, int try_bundle_end_p,
8722                      int only_bundle_end_p)
8723 {
8724   struct bundle_state *curr_state;
8725
8726   curr_state = get_free_bundle_state ();
8727   memcpy (curr_state->dfa_state, originator->dfa_state, dfa_state_size);
8728   curr_state->insn = insn;
8729   curr_state->insn_num = originator->insn_num + 1;
8730   curr_state->cost = originator->cost;
8731   curr_state->originator = originator;
8732   curr_state->before_nops_num = before_nops_num;
8733   curr_state->after_nops_num = 0;
8734   curr_state->accumulated_insns_num
8735     = originator->accumulated_insns_num + before_nops_num;
8736   curr_state->branch_deviation = originator->branch_deviation;
8737   curr_state->middle_bundle_stops = originator->middle_bundle_stops;
8738   gcc_assert (insn);
8739   if (INSN_CODE (insn) == CODE_FOR_insn_group_barrier)
8740     {
8741       gcc_assert (GET_MODE (insn) != TImode);
8742       if (!try_issue_nops (curr_state, before_nops_num))
8743         return;
8744       if (!try_issue_insn (curr_state, insn))
8745         return;
8746       memcpy (temp_dfa_state, curr_state->dfa_state, dfa_state_size);
8747       if (curr_state->accumulated_insns_num % 3 != 0)
8748         curr_state->middle_bundle_stops++;
8749       if (state_transition (temp_dfa_state, dfa_pre_cycle_insn) >= 0
8750           && curr_state->accumulated_insns_num % 3 != 0)
8751         {
8752           free_bundle_state (curr_state);
8753           return;
8754         }
8755     }
8756   else if (GET_MODE (insn) != TImode)
8757     {
8758       if (!try_issue_nops (curr_state, before_nops_num))
8759         return;
8760       if (!try_issue_insn (curr_state, insn))
8761         return;
8762       curr_state->accumulated_insns_num++;
8763       gcc_assert (!unknown_for_bundling_p (insn));
8764
8765       if (ia64_safe_type (insn) == TYPE_L)
8766         curr_state->accumulated_insns_num++;
8767     }
8768   else
8769     {
8770       /* If this is an insn that must be first in a group, then don't allow
8771          nops to be emitted before it.  Currently, alloc is the only such
8772          supported instruction.  */
8773       /* ??? The bundling automatons should handle this for us, but they do
8774          not yet have support for the first_insn attribute.  */
8775       if (before_nops_num > 0 && get_attr_first_insn (insn) == FIRST_INSN_YES)
8776         {
8777           free_bundle_state (curr_state);
8778           return;
8779         }
8780
8781       state_transition (curr_state->dfa_state, dfa_pre_cycle_insn);
8782       state_transition (curr_state->dfa_state, NULL);
8783       curr_state->cost++;
8784       if (!try_issue_nops (curr_state, before_nops_num))
8785         return;
8786       if (!try_issue_insn (curr_state, insn))
8787         return;
8788       curr_state->accumulated_insns_num++;
8789       if (unknown_for_bundling_p (insn))
8790         {
8791           /* Finish bundle containing asm insn.  */
8792           curr_state->after_nops_num
8793             = 3 - curr_state->accumulated_insns_num % 3;
8794           curr_state->accumulated_insns_num
8795             += 3 - curr_state->accumulated_insns_num % 3;
8796         }
8797       else if (ia64_safe_type (insn) == TYPE_L)
8798         curr_state->accumulated_insns_num++;
8799     }
8800   if (ia64_safe_type (insn) == TYPE_B)
8801     curr_state->branch_deviation
8802       += 2 - (curr_state->accumulated_insns_num - 1) % 3;
8803   if (try_bundle_end_p && curr_state->accumulated_insns_num % 3 != 0)
8804     {
8805       if (!only_bundle_end_p && insert_bundle_state (curr_state))
8806         {
8807           state_t dfa_state;
8808           struct bundle_state *curr_state1;
8809           struct bundle_state *allocated_states_chain;
8810
8811           curr_state1 = get_free_bundle_state ();
8812           dfa_state = curr_state1->dfa_state;
8813           allocated_states_chain = curr_state1->allocated_states_chain;
8814           *curr_state1 = *curr_state;
8815           curr_state1->dfa_state = dfa_state;
8816           curr_state1->allocated_states_chain = allocated_states_chain;
8817           memcpy (curr_state1->dfa_state, curr_state->dfa_state,
8818                   dfa_state_size);
8819           curr_state = curr_state1;
8820         }
8821       if (!try_issue_nops (curr_state,
8822                            3 - curr_state->accumulated_insns_num % 3))
8823         return;
8824       curr_state->after_nops_num
8825         = 3 - curr_state->accumulated_insns_num % 3;
8826       curr_state->accumulated_insns_num
8827         += 3 - curr_state->accumulated_insns_num % 3;
8828     }
8829   if (!insert_bundle_state (curr_state))
8830     free_bundle_state (curr_state);
8831   return;
8832 }
8833
8834 /* The following function returns position in the two window bundle
8835    for given STATE.  */
8836
8837 static int
8838 get_max_pos (state_t state)
8839 {
8840   if (cpu_unit_reservation_p (state, pos_6))
8841     return 6;
8842   else if (cpu_unit_reservation_p (state, pos_5))
8843     return 5;
8844   else if (cpu_unit_reservation_p (state, pos_4))
8845     return 4;
8846   else if (cpu_unit_reservation_p (state, pos_3))
8847     return 3;
8848   else if (cpu_unit_reservation_p (state, pos_2))
8849     return 2;
8850   else if (cpu_unit_reservation_p (state, pos_1))
8851     return 1;
8852   else
8853     return 0;
8854 }
8855
8856 /* The function returns code of a possible template for given position
8857    and state.  The function should be called only with 2 values of
8858    position equal to 3 or 6.  We avoid generating F NOPs by putting
8859    templates containing F insns at the end of the template search
8860    because undocumented anomaly in McKinley derived cores which can
8861    cause stalls if an F-unit insn (including a NOP) is issued within a
8862    six-cycle window after reading certain application registers (such
8863    as ar.bsp).  Furthermore, power-considerations also argue against
8864    the use of F-unit instructions unless they're really needed.  */
8865
8866 static int
8867 get_template (state_t state, int pos)
8868 {
8869   switch (pos)
8870     {
8871     case 3:
8872       if (cpu_unit_reservation_p (state, _0mmi_))
8873         return 1;
8874       else if (cpu_unit_reservation_p (state, _0mii_))
8875         return 0;
8876       else if (cpu_unit_reservation_p (state, _0mmb_))
8877         return 7;
8878       else if (cpu_unit_reservation_p (state, _0mib_))
8879         return 6;
8880       else if (cpu_unit_reservation_p (state, _0mbb_))
8881         return 5;
8882       else if (cpu_unit_reservation_p (state, _0bbb_))
8883         return 4;
8884       else if (cpu_unit_reservation_p (state, _0mmf_))
8885         return 3;
8886       else if (cpu_unit_reservation_p (state, _0mfi_))
8887         return 2;
8888       else if (cpu_unit_reservation_p (state, _0mfb_))
8889         return 8;
8890       else if (cpu_unit_reservation_p (state, _0mlx_))
8891         return 9;
8892       else
8893         gcc_unreachable ();
8894     case 6:
8895       if (cpu_unit_reservation_p (state, _1mmi_))
8896         return 1;
8897       else if (cpu_unit_reservation_p (state, _1mii_))
8898         return 0;
8899       else if (cpu_unit_reservation_p (state, _1mmb_))
8900         return 7;
8901       else if (cpu_unit_reservation_p (state, _1mib_))
8902         return 6;
8903       else if (cpu_unit_reservation_p (state, _1mbb_))
8904         return 5;
8905       else if (cpu_unit_reservation_p (state, _1bbb_))
8906         return 4;
8907       else if (_1mmf_ >= 0 && cpu_unit_reservation_p (state, _1mmf_))
8908         return 3;
8909       else if (cpu_unit_reservation_p (state, _1mfi_))
8910         return 2;
8911       else if (cpu_unit_reservation_p (state, _1mfb_))
8912         return 8;
8913       else if (cpu_unit_reservation_p (state, _1mlx_))
8914         return 9;
8915       else
8916         gcc_unreachable ();
8917     default:
8918       gcc_unreachable ();
8919     }
8920 }
8921
8922 /* True when INSN is important for bundling.  */
8923
8924 static bool
8925 important_for_bundling_p (rtx_insn *insn)
8926 {
8927   return (INSN_P (insn)
8928           && ia64_safe_itanium_class (insn) != ITANIUM_CLASS_IGNORE
8929           && GET_CODE (PATTERN (insn)) != USE
8930           && GET_CODE (PATTERN (insn)) != CLOBBER);
8931 }
8932
8933 /* The following function returns an insn important for insn bundling
8934    followed by INSN and before TAIL.  */
8935
8936 static rtx_insn *
8937 get_next_important_insn (rtx_insn *insn, rtx_insn *tail)
8938 {
8939   for (; insn && insn != tail; insn = NEXT_INSN (insn))
8940     if (important_for_bundling_p (insn))
8941       return insn;
8942   return NULL;
8943 }
8944
8945 /* True when INSN is unknown, but important, for bundling.  */
8946
8947 static bool
8948 unknown_for_bundling_p (rtx_insn *insn)
8949 {
8950   return (INSN_P (insn)
8951           && ia64_safe_itanium_class (insn) == ITANIUM_CLASS_UNKNOWN
8952           && GET_CODE (PATTERN (insn)) != USE
8953           && GET_CODE (PATTERN (insn)) != CLOBBER);
8954 }
8955
8956 /* Add a bundle selector TEMPLATE0 before INSN.  */
8957
8958 static void
8959 ia64_add_bundle_selector_before (int template0, rtx_insn *insn)
8960 {
8961   rtx b = gen_bundle_selector (GEN_INT (template0));
8962
8963   ia64_emit_insn_before (b, insn);
8964 #if NR_BUNDLES == 10
8965   if ((template0 == 4 || template0 == 5)
8966       && ia64_except_unwind_info (&global_options) == UI_TARGET)
8967     {
8968       int i;
8969       rtx note = NULL_RTX;
8970
8971       /* In .mbb and .bbb bundles, check if CALL_INSN isn't in the
8972          first or second slot.  If it is and has REG_EH_NOTE set, copy it
8973          to following nops, as br.call sets rp to the address of following
8974          bundle and therefore an EH region end must be on a bundle
8975          boundary.  */
8976       insn = PREV_INSN (insn);
8977       for (i = 0; i < 3; i++)
8978         {
8979           do
8980             insn = next_active_insn (insn);
8981           while (NONJUMP_INSN_P (insn)
8982                  && get_attr_empty (insn) == EMPTY_YES);
8983           if (CALL_P (insn))
8984             note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
8985           else if (note)
8986             {
8987               int code;
8988
8989               gcc_assert ((code = recog_memoized (insn)) == CODE_FOR_nop
8990                           || code == CODE_FOR_nop_b);
8991               if (find_reg_note (insn, REG_EH_REGION, NULL_RTX))
8992                 note = NULL_RTX;
8993               else
8994                 add_reg_note (insn, REG_EH_REGION, XEXP (note, 0));
8995             }
8996         }
8997     }
8998 #endif
8999 }
9000
9001 /* The following function does insn bundling.  Bundling means
9002    inserting templates and nop insns to fit insn groups into permitted
9003    templates.  Instruction scheduling uses NDFA (non-deterministic
9004    finite automata) encoding informations about the templates and the
9005    inserted nops.  Nondeterminism of the automata permits follows
9006    all possible insn sequences very fast.
9007
9008    Unfortunately it is not possible to get information about inserting
9009    nop insns and used templates from the automata states.  The
9010    automata only says that we can issue an insn possibly inserting
9011    some nops before it and using some template.  Therefore insn
9012    bundling in this function is implemented by using DFA
9013    (deterministic finite automata).  We follow all possible insn
9014    sequences by inserting 0-2 nops (that is what the NDFA describe for
9015    insn scheduling) before/after each insn being bundled.  We know the
9016    start of simulated processor cycle from insn scheduling (insn
9017    starting a new cycle has TImode).
9018
9019    Simple implementation of insn bundling would create enormous
9020    number of possible insn sequences satisfying information about new
9021    cycle ticks taken from the insn scheduling.  To make the algorithm
9022    practical we use dynamic programming.  Each decision (about
9023    inserting nops and implicitly about previous decisions) is described
9024    by structure bundle_state (see above).  If we generate the same
9025    bundle state (key is automaton state after issuing the insns and
9026    nops for it), we reuse already generated one.  As consequence we
9027    reject some decisions which cannot improve the solution and
9028    reduce memory for the algorithm.
9029
9030    When we reach the end of EBB (extended basic block), we choose the
9031    best sequence and then, moving back in EBB, insert templates for
9032    the best alternative.  The templates are taken from querying
9033    automaton state for each insn in chosen bundle states.
9034
9035    So the algorithm makes two (forward and backward) passes through
9036    EBB.  */
9037
9038 static void
9039 bundling (FILE *dump, int verbose, rtx_insn *prev_head_insn, rtx_insn *tail)
9040 {
9041   struct bundle_state *curr_state, *next_state, *best_state;
9042   rtx_insn *insn, *next_insn;
9043   int insn_num;
9044   int i, bundle_end_p, only_bundle_end_p, asm_p;
9045   int pos = 0, max_pos, template0, template1;
9046   rtx_insn *b;
9047   enum attr_type type;
9048
9049   insn_num = 0;
9050   /* Count insns in the EBB.  */
9051   for (insn = NEXT_INSN (prev_head_insn);
9052        insn && insn != tail;
9053        insn = NEXT_INSN (insn))
9054     if (INSN_P (insn))
9055       insn_num++;
9056   if (insn_num == 0)
9057     return;
9058   bundling_p = 1;
9059   dfa_clean_insn_cache ();
9060   initiate_bundle_state_table ();
9061   index_to_bundle_states = XNEWVEC (struct bundle_state *, insn_num + 2);
9062   /* First (forward) pass -- generation of bundle states.  */
9063   curr_state = get_free_bundle_state ();
9064   curr_state->insn = NULL;
9065   curr_state->before_nops_num = 0;
9066   curr_state->after_nops_num = 0;
9067   curr_state->insn_num = 0;
9068   curr_state->cost = 0;
9069   curr_state->accumulated_insns_num = 0;
9070   curr_state->branch_deviation = 0;
9071   curr_state->middle_bundle_stops = 0;
9072   curr_state->next = NULL;
9073   curr_state->originator = NULL;
9074   state_reset (curr_state->dfa_state);
9075   index_to_bundle_states [0] = curr_state;
9076   insn_num = 0;
9077   /* Shift cycle mark if it is put on insn which could be ignored.  */
9078   for (insn = NEXT_INSN (prev_head_insn);
9079        insn != tail;
9080        insn = NEXT_INSN (insn))
9081     if (INSN_P (insn)
9082         && !important_for_bundling_p (insn)
9083         && GET_MODE (insn) == TImode)
9084       {
9085         PUT_MODE (insn, VOIDmode);
9086         for (next_insn = NEXT_INSN (insn);
9087              next_insn != tail;
9088              next_insn = NEXT_INSN (next_insn))
9089           if (important_for_bundling_p (next_insn)
9090               && INSN_CODE (next_insn) != CODE_FOR_insn_group_barrier)
9091             {
9092               PUT_MODE (next_insn, TImode);
9093               break;
9094             }
9095       }
9096   /* Forward pass: generation of bundle states.  */
9097   for (insn = get_next_important_insn (NEXT_INSN (prev_head_insn), tail);
9098        insn != NULL_RTX;
9099        insn = next_insn)
9100     {
9101       gcc_assert (important_for_bundling_p (insn));
9102       type = ia64_safe_type (insn);
9103       next_insn = get_next_important_insn (NEXT_INSN (insn), tail);
9104       insn_num++;
9105       index_to_bundle_states [insn_num] = NULL;
9106       for (curr_state = index_to_bundle_states [insn_num - 1];
9107            curr_state != NULL;
9108            curr_state = next_state)
9109         {
9110           pos = curr_state->accumulated_insns_num % 3;
9111           next_state = curr_state->next;
9112           /* We must fill up the current bundle in order to start a
9113              subsequent asm insn in a new bundle.  Asm insn is always
9114              placed in a separate bundle.  */
9115           only_bundle_end_p
9116             = (next_insn != NULL_RTX
9117                && INSN_CODE (insn) == CODE_FOR_insn_group_barrier
9118                && unknown_for_bundling_p (next_insn));
9119           /* We may fill up the current bundle if it is the cycle end
9120              without a group barrier.  */
9121           bundle_end_p
9122             = (only_bundle_end_p || next_insn == NULL_RTX
9123                || (GET_MODE (next_insn) == TImode
9124                    && INSN_CODE (insn) != CODE_FOR_insn_group_barrier));
9125           if (type == TYPE_F || type == TYPE_B || type == TYPE_L
9126               || type == TYPE_S)
9127             issue_nops_and_insn (curr_state, 2, insn, bundle_end_p,
9128                                  only_bundle_end_p);
9129           issue_nops_and_insn (curr_state, 1, insn, bundle_end_p,
9130                                only_bundle_end_p);
9131           issue_nops_and_insn (curr_state, 0, insn, bundle_end_p,
9132                                only_bundle_end_p);
9133         }
9134       gcc_assert (index_to_bundle_states [insn_num]);
9135       for (curr_state = index_to_bundle_states [insn_num];
9136            curr_state != NULL;
9137            curr_state = curr_state->next)
9138         if (verbose >= 2 && dump)
9139           {
9140             /* This structure is taken from generated code of the
9141                pipeline hazard recognizer (see file insn-attrtab.c).
9142                Please don't forget to change the structure if a new
9143                automaton is added to .md file.  */
9144             struct DFA_chip
9145             {
9146               unsigned short one_automaton_state;
9147               unsigned short oneb_automaton_state;
9148               unsigned short two_automaton_state;
9149               unsigned short twob_automaton_state;
9150             };
9151
9152             fprintf
9153               (dump,
9154                "//    Bundle state %d (orig %d, cost %d, nops %d/%d, insns %d, branch %d, mid.stops %d state %d) for %d\n",
9155                curr_state->unique_num,
9156                (curr_state->originator == NULL
9157                 ? -1 : curr_state->originator->unique_num),
9158                curr_state->cost,
9159                curr_state->before_nops_num, curr_state->after_nops_num,
9160                curr_state->accumulated_insns_num, curr_state->branch_deviation,
9161                curr_state->middle_bundle_stops,
9162                ((struct DFA_chip *) curr_state->dfa_state)->twob_automaton_state,
9163                INSN_UID (insn));
9164           }
9165     }
9166
9167   /* We should find a solution because the 2nd insn scheduling has
9168      found one.  */
9169   gcc_assert (index_to_bundle_states [insn_num]);
9170   /* Find a state corresponding to the best insn sequence.  */
9171   best_state = NULL;
9172   for (curr_state = index_to_bundle_states [insn_num];
9173        curr_state != NULL;
9174        curr_state = curr_state->next)
9175     /* We are just looking at the states with fully filled up last
9176        bundle.  The first we prefer insn sequences with minimal cost
9177        then with minimal inserted nops and finally with branch insns
9178        placed in the 3rd slots.  */
9179     if (curr_state->accumulated_insns_num % 3 == 0
9180         && (best_state == NULL || best_state->cost > curr_state->cost
9181             || (best_state->cost == curr_state->cost
9182                 && (curr_state->accumulated_insns_num
9183                     < best_state->accumulated_insns_num
9184                     || (curr_state->accumulated_insns_num
9185                         == best_state->accumulated_insns_num
9186                         && (curr_state->branch_deviation
9187                             < best_state->branch_deviation
9188                             || (curr_state->branch_deviation
9189                                 == best_state->branch_deviation
9190                                 && curr_state->middle_bundle_stops
9191                                 < best_state->middle_bundle_stops)))))))
9192       best_state = curr_state;
9193   /* Second (backward) pass: adding nops and templates.  */
9194   gcc_assert (best_state);
9195   insn_num = best_state->before_nops_num;
9196   template0 = template1 = -1;
9197   for (curr_state = best_state;
9198        curr_state->originator != NULL;
9199        curr_state = curr_state->originator)
9200     {
9201       insn = curr_state->insn;
9202       asm_p = unknown_for_bundling_p (insn);
9203       insn_num++;
9204       if (verbose >= 2 && dump)
9205         {
9206           struct DFA_chip
9207           {
9208             unsigned short one_automaton_state;
9209             unsigned short oneb_automaton_state;
9210             unsigned short two_automaton_state;
9211             unsigned short twob_automaton_state;
9212           };
9213
9214           fprintf
9215             (dump,
9216              "//    Best %d (orig %d, cost %d, nops %d/%d, insns %d, branch %d, mid.stops %d, state %d) for %d\n",
9217              curr_state->unique_num,
9218              (curr_state->originator == NULL
9219               ? -1 : curr_state->originator->unique_num),
9220              curr_state->cost,
9221              curr_state->before_nops_num, curr_state->after_nops_num,
9222              curr_state->accumulated_insns_num, curr_state->branch_deviation,
9223              curr_state->middle_bundle_stops,
9224              ((struct DFA_chip *) curr_state->dfa_state)->twob_automaton_state,
9225              INSN_UID (insn));
9226         }
9227       /* Find the position in the current bundle window.  The window can
9228          contain at most two bundles.  Two bundle window means that
9229          the processor will make two bundle rotation.  */
9230       max_pos = get_max_pos (curr_state->dfa_state);
9231       if (max_pos == 6
9232           /* The following (negative template number) means that the
9233              processor did one bundle rotation.  */
9234           || (max_pos == 3 && template0 < 0))
9235         {
9236           /* We are at the end of the window -- find template(s) for
9237              its bundle(s).  */
9238           pos = max_pos;
9239           if (max_pos == 3)
9240             template0 = get_template (curr_state->dfa_state, 3);
9241           else
9242             {
9243               template1 = get_template (curr_state->dfa_state, 3);
9244               template0 = get_template (curr_state->dfa_state, 6);
9245             }
9246         }
9247       if (max_pos > 3 && template1 < 0)
9248         /* It may happen when we have the stop inside a bundle.  */
9249         {
9250           gcc_assert (pos <= 3);
9251           template1 = get_template (curr_state->dfa_state, 3);
9252           pos += 3;
9253         }
9254       if (!asm_p)
9255         /* Emit nops after the current insn.  */
9256         for (i = 0; i < curr_state->after_nops_num; i++)
9257           {
9258             rtx nop_pat = gen_nop ();
9259             rtx_insn *nop = emit_insn_after (nop_pat, insn);
9260             pos--;
9261             gcc_assert (pos >= 0);
9262             if (pos % 3 == 0)
9263               {
9264                 /* We are at the start of a bundle: emit the template
9265                    (it should be defined).  */
9266                 gcc_assert (template0 >= 0);
9267                 ia64_add_bundle_selector_before (template0, nop);
9268                 /* If we have two bundle window, we make one bundle
9269                    rotation.  Otherwise template0 will be undefined
9270                    (negative value).  */
9271                 template0 = template1;
9272                 template1 = -1;
9273               }
9274           }
9275       /* Move the position backward in the window.  Group barrier has
9276          no slot.  Asm insn takes all bundle.  */
9277       if (INSN_CODE (insn) != CODE_FOR_insn_group_barrier
9278           && !unknown_for_bundling_p (insn))
9279         pos--;
9280       /* Long insn takes 2 slots.  */
9281       if (ia64_safe_type (insn) == TYPE_L)
9282         pos--;
9283       gcc_assert (pos >= 0);
9284       if (pos % 3 == 0
9285           && INSN_CODE (insn) != CODE_FOR_insn_group_barrier
9286           && !unknown_for_bundling_p (insn))
9287         {
9288           /* The current insn is at the bundle start: emit the
9289              template.  */
9290           gcc_assert (template0 >= 0);
9291           ia64_add_bundle_selector_before (template0, insn);
9292           b = PREV_INSN (insn);
9293           insn = b;
9294           /* See comment above in analogous place for emitting nops
9295              after the insn.  */
9296           template0 = template1;
9297           template1 = -1;
9298         }
9299       /* Emit nops after the current insn.  */
9300       for (i = 0; i < curr_state->before_nops_num; i++)
9301         {
9302           rtx nop_pat = gen_nop ();
9303           ia64_emit_insn_before (nop_pat, insn);
9304           rtx_insn *nop = PREV_INSN (insn);
9305           insn = nop;
9306           pos--;
9307           gcc_assert (pos >= 0);
9308           if (pos % 3 == 0)
9309             {
9310               /* See comment above in analogous place for emitting nops
9311                  after the insn.  */
9312               gcc_assert (template0 >= 0);
9313               ia64_add_bundle_selector_before (template0, insn);
9314               b = PREV_INSN (insn);
9315               insn = b;
9316               template0 = template1;
9317               template1 = -1;
9318             }
9319         }
9320     }
9321
9322 #ifdef ENABLE_CHECKING
9323   {
9324     /* Assert right calculation of middle_bundle_stops.  */
9325     int num = best_state->middle_bundle_stops;
9326     bool start_bundle = true, end_bundle = false;
9327
9328     for (insn = NEXT_INSN (prev_head_insn);
9329          insn && insn != tail;
9330          insn = NEXT_INSN (insn))
9331       {
9332         if (!INSN_P (insn))
9333           continue;
9334         if (recog_memoized (insn) == CODE_FOR_bundle_selector)
9335           start_bundle = true;
9336         else
9337           {
9338             rtx_insn *next_insn;
9339
9340             for (next_insn = NEXT_INSN (insn);
9341                  next_insn && next_insn != tail;
9342                  next_insn = NEXT_INSN (next_insn))
9343               if (INSN_P (next_insn)
9344                   && (ia64_safe_itanium_class (next_insn)
9345                       != ITANIUM_CLASS_IGNORE
9346                       || recog_memoized (next_insn)
9347                       == CODE_FOR_bundle_selector)
9348                   && GET_CODE (PATTERN (next_insn)) != USE
9349                   && GET_CODE (PATTERN (next_insn)) != CLOBBER)
9350                 break;
9351
9352             end_bundle = next_insn == NULL_RTX
9353              || next_insn == tail
9354              || (INSN_P (next_insn)
9355                  && recog_memoized (next_insn)
9356                  == CODE_FOR_bundle_selector);
9357             if (recog_memoized (insn) == CODE_FOR_insn_group_barrier
9358                 && !start_bundle && !end_bundle
9359                 && next_insn
9360                 && !unknown_for_bundling_p (next_insn))
9361               num--;
9362
9363             start_bundle = false;
9364           }
9365       }
9366
9367     gcc_assert (num == 0);
9368   }
9369 #endif
9370
9371   free (index_to_bundle_states);
9372   finish_bundle_state_table ();
9373   bundling_p = 0;
9374   dfa_clean_insn_cache ();
9375 }
9376
9377 /* The following function is called at the end of scheduling BB or
9378    EBB.  After reload, it inserts stop bits and does insn bundling.  */
9379
9380 static void
9381 ia64_sched_finish (FILE *dump, int sched_verbose)
9382 {
9383   if (sched_verbose)
9384     fprintf (dump, "// Finishing schedule.\n");
9385   if (!reload_completed)
9386     return;
9387   if (reload_completed)
9388     {
9389       final_emit_insn_group_barriers (dump);
9390       bundling (dump, sched_verbose, current_sched_info->prev_head,
9391                 current_sched_info->next_tail);
9392       if (sched_verbose && dump)
9393         fprintf (dump, "//    finishing %d-%d\n",
9394                  INSN_UID (NEXT_INSN (current_sched_info->prev_head)),
9395                  INSN_UID (PREV_INSN (current_sched_info->next_tail)));
9396
9397       return;
9398     }
9399 }
9400
9401 /* The following function inserts stop bits in scheduled BB or EBB.  */
9402
9403 static void
9404 final_emit_insn_group_barriers (FILE *dump ATTRIBUTE_UNUSED)
9405 {
9406   rtx_insn *insn;
9407   int need_barrier_p = 0;
9408   int seen_good_insn = 0;
9409
9410   init_insn_group_barriers ();
9411
9412   for (insn = NEXT_INSN (current_sched_info->prev_head);
9413        insn != current_sched_info->next_tail;
9414        insn = NEXT_INSN (insn))
9415     {
9416       if (BARRIER_P (insn))
9417         {
9418           rtx_insn *last = prev_active_insn (insn);
9419
9420           if (! last)
9421             continue;
9422           if (JUMP_TABLE_DATA_P (last))
9423             last = prev_active_insn (last);
9424           if (recog_memoized (last) != CODE_FOR_insn_group_barrier)
9425             emit_insn_after (gen_insn_group_barrier (GEN_INT (3)), last);
9426
9427           init_insn_group_barriers ();
9428           seen_good_insn = 0;
9429           need_barrier_p = 0;
9430         }
9431       else if (NONDEBUG_INSN_P (insn))
9432         {
9433           if (recog_memoized (insn) == CODE_FOR_insn_group_barrier)
9434             {
9435               init_insn_group_barriers ();
9436               seen_good_insn = 0;
9437               need_barrier_p = 0;
9438             }
9439           else if (need_barrier_p || group_barrier_needed (insn)
9440                    || (mflag_sched_stop_bits_after_every_cycle
9441                        && GET_MODE (insn) == TImode
9442                        && seen_good_insn))
9443             {
9444               if (TARGET_EARLY_STOP_BITS)
9445                 {
9446                   rtx_insn *last;
9447
9448                   for (last = insn;
9449                        last != current_sched_info->prev_head;
9450                        last = PREV_INSN (last))
9451                     if (INSN_P (last) && GET_MODE (last) == TImode
9452                         && stops_p [INSN_UID (last)])
9453                       break;
9454                   if (last == current_sched_info->prev_head)
9455                     last = insn;
9456                   last = prev_active_insn (last);
9457                   if (last
9458                       && recog_memoized (last) != CODE_FOR_insn_group_barrier)
9459                     emit_insn_after (gen_insn_group_barrier (GEN_INT (3)),
9460                                      last);
9461                   init_insn_group_barriers ();
9462                   for (last = NEXT_INSN (last);
9463                        last != insn;
9464                        last = NEXT_INSN (last))
9465                     if (INSN_P (last))
9466                       {
9467                         group_barrier_needed (last);
9468                         if (recog_memoized (last) >= 0
9469                             && important_for_bundling_p (last))
9470                           seen_good_insn = 1;
9471                       }
9472                 }
9473               else
9474                 {
9475                   emit_insn_before (gen_insn_group_barrier (GEN_INT (3)),
9476                                     insn);
9477                   init_insn_group_barriers ();
9478                   seen_good_insn = 0;
9479                 }
9480               group_barrier_needed (insn);
9481               if (recog_memoized (insn) >= 0
9482                   && important_for_bundling_p (insn))
9483                 seen_good_insn = 1;
9484             }
9485           else if (recog_memoized (insn) >= 0
9486                    && important_for_bundling_p (insn))
9487             seen_good_insn = 1;
9488           need_barrier_p = (CALL_P (insn) || unknown_for_bundling_p (insn));
9489         }
9490     }
9491 }
9492
9493 \f
9494
9495 /* If the following function returns TRUE, we will use the DFA
9496    insn scheduler.  */
9497
9498 static int
9499 ia64_first_cycle_multipass_dfa_lookahead (void)
9500 {
9501   return (reload_completed ? 6 : 4);
9502 }
9503
9504 /* The following function initiates variable `dfa_pre_cycle_insn'.  */
9505
9506 static void
9507 ia64_init_dfa_pre_cycle_insn (void)
9508 {
9509   if (temp_dfa_state == NULL)
9510     {
9511       dfa_state_size = state_size ();
9512       temp_dfa_state = xmalloc (dfa_state_size);
9513       prev_cycle_state = xmalloc (dfa_state_size);
9514     }
9515   dfa_pre_cycle_insn = make_insn_raw (gen_pre_cycle ());
9516   SET_PREV_INSN (dfa_pre_cycle_insn) = SET_NEXT_INSN (dfa_pre_cycle_insn) = NULL_RTX;
9517   recog_memoized (dfa_pre_cycle_insn);
9518   dfa_stop_insn = make_insn_raw (gen_insn_group_barrier (GEN_INT (3)));
9519   SET_PREV_INSN (dfa_stop_insn) = SET_NEXT_INSN (dfa_stop_insn) = NULL_RTX;
9520   recog_memoized (dfa_stop_insn);
9521 }
9522
9523 /* The following function returns the pseudo insn DFA_PRE_CYCLE_INSN
9524    used by the DFA insn scheduler.  */
9525
9526 static rtx
9527 ia64_dfa_pre_cycle_insn (void)
9528 {
9529   return dfa_pre_cycle_insn;
9530 }
9531
9532 /* The following function returns TRUE if PRODUCER (of type ilog or
9533    ld) produces address for CONSUMER (of type st or stf). */
9534
9535 int
9536 ia64_st_address_bypass_p (rtx_insn *producer, rtx_insn *consumer)
9537 {
9538   rtx dest, reg, mem;
9539
9540   gcc_assert (producer && consumer);
9541   dest = ia64_single_set (producer);
9542   gcc_assert (dest);
9543   reg = SET_DEST (dest);
9544   gcc_assert (reg);
9545   if (GET_CODE (reg) == SUBREG)
9546     reg = SUBREG_REG (reg);
9547   gcc_assert (GET_CODE (reg) == REG);
9548
9549   dest = ia64_single_set (consumer);
9550   gcc_assert (dest);
9551   mem = SET_DEST (dest);
9552   gcc_assert (mem && GET_CODE (mem) == MEM);
9553   return reg_mentioned_p (reg, mem);
9554 }
9555
9556 /* The following function returns TRUE if PRODUCER (of type ilog or
9557    ld) produces address for CONSUMER (of type ld or fld). */
9558
9559 int
9560 ia64_ld_address_bypass_p (rtx_insn *producer, rtx_insn *consumer)
9561 {
9562   rtx dest, src, reg, mem;
9563
9564   gcc_assert (producer && consumer);
9565   dest = ia64_single_set (producer);
9566   gcc_assert (dest);
9567   reg = SET_DEST (dest);
9568   gcc_assert (reg);
9569   if (GET_CODE (reg) == SUBREG)
9570     reg = SUBREG_REG (reg);
9571   gcc_assert (GET_CODE (reg) == REG);
9572
9573   src = ia64_single_set (consumer);
9574   gcc_assert (src);
9575   mem = SET_SRC (src);
9576   gcc_assert (mem);
9577
9578   if (GET_CODE (mem) == UNSPEC && XVECLEN (mem, 0) > 0)
9579     mem = XVECEXP (mem, 0, 0);
9580   else if (GET_CODE (mem) == IF_THEN_ELSE)
9581     /* ??? Is this bypass necessary for ld.c?  */
9582     {
9583       gcc_assert (XINT (XEXP (XEXP (mem, 0), 0), 1) == UNSPEC_LDCCLR);
9584       mem = XEXP (mem, 1);
9585     }
9586
9587   while (GET_CODE (mem) == SUBREG || GET_CODE (mem) == ZERO_EXTEND)
9588     mem = XEXP (mem, 0);
9589
9590   if (GET_CODE (mem) == UNSPEC)
9591     {
9592       int c = XINT (mem, 1);
9593
9594       gcc_assert (c == UNSPEC_LDA || c == UNSPEC_LDS || c == UNSPEC_LDS_A
9595                   || c == UNSPEC_LDSA);
9596       mem = XVECEXP (mem, 0, 0);
9597     }
9598
9599   /* Note that LO_SUM is used for GOT loads.  */
9600   gcc_assert (GET_CODE (mem) == LO_SUM || GET_CODE (mem) == MEM);
9601
9602   return reg_mentioned_p (reg, mem);
9603 }
9604
9605 /* The following function returns TRUE if INSN produces address for a
9606    load/store insn.  We will place such insns into M slot because it
9607    decreases its latency time.  */
9608
9609 int
9610 ia64_produce_address_p (rtx insn)
9611 {
9612   return insn->call;
9613 }
9614
9615 \f
9616 /* Emit pseudo-ops for the assembler to describe predicate relations.
9617    At present this assumes that we only consider predicate pairs to
9618    be mutex, and that the assembler can deduce proper values from
9619    straight-line code.  */
9620
9621 static void
9622 emit_predicate_relation_info (void)
9623 {
9624   basic_block bb;
9625
9626   FOR_EACH_BB_REVERSE_FN (bb, cfun)
9627     {
9628       int r;
9629       rtx_insn *head = BB_HEAD (bb);
9630
9631       /* We only need such notes at code labels.  */
9632       if (! LABEL_P (head))
9633         continue;
9634       if (NOTE_INSN_BASIC_BLOCK_P (NEXT_INSN (head)))
9635         head = NEXT_INSN (head);
9636
9637       /* Skip p0, which may be thought to be live due to (reg:DI p0)
9638          grabbing the entire block of predicate registers.  */
9639       for (r = PR_REG (2); r < PR_REG (64); r += 2)
9640         if (REGNO_REG_SET_P (df_get_live_in (bb), r))
9641           {
9642             rtx p = gen_rtx_REG (BImode, r);
9643             rtx_insn *n = emit_insn_after (gen_pred_rel_mutex (p), head);
9644             if (head == BB_END (bb))
9645               BB_END (bb) = n;
9646             head = n;
9647           }
9648     }
9649
9650   /* Look for conditional calls that do not return, and protect predicate
9651      relations around them.  Otherwise the assembler will assume the call
9652      returns, and complain about uses of call-clobbered predicates after
9653      the call.  */
9654   FOR_EACH_BB_REVERSE_FN (bb, cfun)
9655     {
9656       rtx_insn *insn = BB_HEAD (bb);
9657
9658       while (1)
9659         {
9660           if (CALL_P (insn)
9661               && GET_CODE (PATTERN (insn)) == COND_EXEC
9662               && find_reg_note (insn, REG_NORETURN, NULL_RTX))
9663             {
9664               rtx_insn *b =
9665                 emit_insn_before (gen_safe_across_calls_all (), insn);
9666               rtx_insn *a = emit_insn_after (gen_safe_across_calls_normal (), insn);
9667               if (BB_HEAD (bb) == insn)
9668                 BB_HEAD (bb) = b;
9669               if (BB_END (bb) == insn)
9670                 BB_END (bb) = a;
9671             }
9672
9673           if (insn == BB_END (bb))
9674             break;
9675           insn = NEXT_INSN (insn);
9676         }
9677     }
9678 }
9679
9680 /* Perform machine dependent operations on the rtl chain INSNS.  */
9681
9682 static void
9683 ia64_reorg (void)
9684 {
9685   /* We are freeing block_for_insn in the toplev to keep compatibility
9686      with old MDEP_REORGS that are not CFG based.  Recompute it now.  */
9687   compute_bb_for_insn ();
9688
9689   /* If optimizing, we'll have split before scheduling.  */
9690   if (optimize == 0)
9691     split_all_insns ();
9692
9693   if (optimize && flag_schedule_insns_after_reload
9694       && dbg_cnt (ia64_sched2))
9695     {
9696       basic_block bb;
9697       timevar_push (TV_SCHED2);
9698       ia64_final_schedule = 1;
9699
9700       /* We can't let modulo-sched prevent us from scheduling any bbs,
9701          since we need the final schedule to produce bundle information.  */
9702       FOR_EACH_BB_FN (bb, cfun)
9703         bb->flags &= ~BB_DISABLE_SCHEDULE;
9704
9705       initiate_bundle_states ();
9706       ia64_nop = make_insn_raw (gen_nop ());
9707       SET_PREV_INSN (ia64_nop) = SET_NEXT_INSN (ia64_nop) = NULL_RTX;
9708       recog_memoized (ia64_nop);
9709       clocks_length = get_max_uid () + 1;
9710       stops_p = XCNEWVEC (char, clocks_length);
9711
9712       if (ia64_tune == PROCESSOR_ITANIUM2)
9713         {
9714           pos_1 = get_cpu_unit_code ("2_1");
9715           pos_2 = get_cpu_unit_code ("2_2");
9716           pos_3 = get_cpu_unit_code ("2_3");
9717           pos_4 = get_cpu_unit_code ("2_4");
9718           pos_5 = get_cpu_unit_code ("2_5");
9719           pos_6 = get_cpu_unit_code ("2_6");
9720           _0mii_ = get_cpu_unit_code ("2b_0mii.");
9721           _0mmi_ = get_cpu_unit_code ("2b_0mmi.");
9722           _0mfi_ = get_cpu_unit_code ("2b_0mfi.");
9723           _0mmf_ = get_cpu_unit_code ("2b_0mmf.");
9724           _0bbb_ = get_cpu_unit_code ("2b_0bbb.");
9725           _0mbb_ = get_cpu_unit_code ("2b_0mbb.");
9726           _0mib_ = get_cpu_unit_code ("2b_0mib.");
9727           _0mmb_ = get_cpu_unit_code ("2b_0mmb.");
9728           _0mfb_ = get_cpu_unit_code ("2b_0mfb.");
9729           _0mlx_ = get_cpu_unit_code ("2b_0mlx.");
9730           _1mii_ = get_cpu_unit_code ("2b_1mii.");
9731           _1mmi_ = get_cpu_unit_code ("2b_1mmi.");
9732           _1mfi_ = get_cpu_unit_code ("2b_1mfi.");
9733           _1mmf_ = get_cpu_unit_code ("2b_1mmf.");
9734           _1bbb_ = get_cpu_unit_code ("2b_1bbb.");
9735           _1mbb_ = get_cpu_unit_code ("2b_1mbb.");
9736           _1mib_ = get_cpu_unit_code ("2b_1mib.");
9737           _1mmb_ = get_cpu_unit_code ("2b_1mmb.");
9738           _1mfb_ = get_cpu_unit_code ("2b_1mfb.");
9739           _1mlx_ = get_cpu_unit_code ("2b_1mlx.");
9740         }
9741       else
9742         {
9743           pos_1 = get_cpu_unit_code ("1_1");
9744           pos_2 = get_cpu_unit_code ("1_2");
9745           pos_3 = get_cpu_unit_code ("1_3");
9746           pos_4 = get_cpu_unit_code ("1_4");
9747           pos_5 = get_cpu_unit_code ("1_5");
9748           pos_6 = get_cpu_unit_code ("1_6");
9749           _0mii_ = get_cpu_unit_code ("1b_0mii.");
9750           _0mmi_ = get_cpu_unit_code ("1b_0mmi.");
9751           _0mfi_ = get_cpu_unit_code ("1b_0mfi.");
9752           _0mmf_ = get_cpu_unit_code ("1b_0mmf.");
9753           _0bbb_ = get_cpu_unit_code ("1b_0bbb.");
9754           _0mbb_ = get_cpu_unit_code ("1b_0mbb.");
9755           _0mib_ = get_cpu_unit_code ("1b_0mib.");
9756           _0mmb_ = get_cpu_unit_code ("1b_0mmb.");
9757           _0mfb_ = get_cpu_unit_code ("1b_0mfb.");
9758           _0mlx_ = get_cpu_unit_code ("1b_0mlx.");
9759           _1mii_ = get_cpu_unit_code ("1b_1mii.");
9760           _1mmi_ = get_cpu_unit_code ("1b_1mmi.");
9761           _1mfi_ = get_cpu_unit_code ("1b_1mfi.");
9762           _1mmf_ = get_cpu_unit_code ("1b_1mmf.");
9763           _1bbb_ = get_cpu_unit_code ("1b_1bbb.");
9764           _1mbb_ = get_cpu_unit_code ("1b_1mbb.");
9765           _1mib_ = get_cpu_unit_code ("1b_1mib.");
9766           _1mmb_ = get_cpu_unit_code ("1b_1mmb.");
9767           _1mfb_ = get_cpu_unit_code ("1b_1mfb.");
9768           _1mlx_ = get_cpu_unit_code ("1b_1mlx.");
9769         }
9770
9771       if (flag_selective_scheduling2
9772           && !maybe_skip_selective_scheduling ())
9773         run_selective_scheduling ();
9774       else
9775         schedule_ebbs ();
9776
9777       /* Redo alignment computation, as it might gone wrong.  */
9778       compute_alignments ();
9779
9780       /* We cannot reuse this one because it has been corrupted by the
9781          evil glat.  */
9782       finish_bundle_states ();
9783       free (stops_p);
9784       stops_p = NULL;
9785       emit_insn_group_barriers (dump_file);
9786
9787       ia64_final_schedule = 0;
9788       timevar_pop (TV_SCHED2);
9789     }
9790   else
9791     emit_all_insn_group_barriers (dump_file);
9792
9793   df_analyze ();
9794
9795   /* A call must not be the last instruction in a function, so that the
9796      return address is still within the function, so that unwinding works
9797      properly.  Note that IA-64 differs from dwarf2 on this point.  */
9798   if (ia64_except_unwind_info (&global_options) == UI_TARGET)
9799     {
9800       rtx_insn *insn;
9801       int saw_stop = 0;
9802
9803       insn = get_last_insn ();
9804       if (! INSN_P (insn))
9805         insn = prev_active_insn (insn);
9806       if (insn)
9807         {
9808           /* Skip over insns that expand to nothing.  */
9809           while (NONJUMP_INSN_P (insn)
9810                  && get_attr_empty (insn) == EMPTY_YES)
9811             {
9812               if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
9813                   && XINT (PATTERN (insn), 1) == UNSPECV_INSN_GROUP_BARRIER)
9814                 saw_stop = 1;
9815               insn = prev_active_insn (insn);
9816             }
9817           if (CALL_P (insn))
9818             {
9819               if (! saw_stop)
9820                 emit_insn (gen_insn_group_barrier (GEN_INT (3)));
9821               emit_insn (gen_break_f ());
9822               emit_insn (gen_insn_group_barrier (GEN_INT (3)));
9823             }
9824         }
9825     }
9826
9827   emit_predicate_relation_info ();
9828
9829   if (flag_var_tracking)
9830     {
9831       timevar_push (TV_VAR_TRACKING);
9832       variable_tracking_main ();
9833       timevar_pop (TV_VAR_TRACKING);
9834     }
9835   df_finish_pass (false);
9836 }
9837 \f
9838 /* Return true if REGNO is used by the epilogue.  */
9839
9840 int
9841 ia64_epilogue_uses (int regno)
9842 {
9843   switch (regno)
9844     {
9845     case R_GR (1):
9846       /* With a call to a function in another module, we will write a new
9847          value to "gp".  After returning from such a call, we need to make
9848          sure the function restores the original gp-value, even if the
9849          function itself does not use the gp anymore.  */
9850       return !(TARGET_AUTO_PIC || TARGET_NO_PIC);
9851
9852     case IN_REG (0): case IN_REG (1): case IN_REG (2): case IN_REG (3):
9853     case IN_REG (4): case IN_REG (5): case IN_REG (6): case IN_REG (7):
9854       /* For functions defined with the syscall_linkage attribute, all
9855          input registers are marked as live at all function exits.  This
9856          prevents the register allocator from using the input registers,
9857          which in turn makes it possible to restart a system call after
9858          an interrupt without having to save/restore the input registers.
9859          This also prevents kernel data from leaking to application code.  */
9860       return lookup_attribute ("syscall_linkage",
9861            TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))) != NULL;
9862
9863     case R_BR (0):
9864       /* Conditional return patterns can't represent the use of `b0' as
9865          the return address, so we force the value live this way.  */
9866       return 1;
9867
9868     case AR_PFS_REGNUM:
9869       /* Likewise for ar.pfs, which is used by br.ret.  */
9870       return 1;
9871
9872     default:
9873       return 0;
9874     }
9875 }
9876
9877 /* Return true if REGNO is used by the frame unwinder.  */
9878
9879 int
9880 ia64_eh_uses (int regno)
9881 {
9882   unsigned int r;
9883
9884   if (! reload_completed)
9885     return 0;
9886
9887   if (regno == 0)
9888     return 0;
9889
9890   for (r = reg_save_b0; r <= reg_save_ar_lc; r++)
9891     if (regno == current_frame_info.r[r]
9892        || regno == emitted_frame_related_regs[r])
9893       return 1;
9894
9895   return 0;
9896 }
9897 \f
9898 /* Return true if this goes in small data/bss.  */
9899
9900 /* ??? We could also support own long data here.  Generating movl/add/ld8
9901    instead of addl,ld8/ld8.  This makes the code bigger, but should make the
9902    code faster because there is one less load.  This also includes incomplete
9903    types which can't go in sdata/sbss.  */
9904
9905 static bool
9906 ia64_in_small_data_p (const_tree exp)
9907 {
9908   if (TARGET_NO_SDATA)
9909     return false;
9910
9911   /* We want to merge strings, so we never consider them small data.  */
9912   if (TREE_CODE (exp) == STRING_CST)
9913     return false;
9914
9915   /* Functions are never small data.  */
9916   if (TREE_CODE (exp) == FUNCTION_DECL)
9917     return false;
9918
9919   if (TREE_CODE (exp) == VAR_DECL && DECL_SECTION_NAME (exp))
9920     {
9921       const char *section = DECL_SECTION_NAME (exp);
9922
9923       if (strcmp (section, ".sdata") == 0
9924           || strncmp (section, ".sdata.", 7) == 0
9925           || strncmp (section, ".gnu.linkonce.s.", 16) == 0
9926           || strcmp (section, ".sbss") == 0
9927           || strncmp (section, ".sbss.", 6) == 0
9928           || strncmp (section, ".gnu.linkonce.sb.", 17) == 0)
9929         return true;
9930     }
9931   else
9932     {
9933       HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
9934
9935       /* If this is an incomplete type with size 0, then we can't put it
9936          in sdata because it might be too big when completed.  */
9937       if (size > 0 && size <= ia64_section_threshold)
9938         return true;
9939     }
9940
9941   return false;
9942 }
9943 \f
9944 /* Output assembly directives for prologue regions.  */
9945
9946 /* The current basic block number.  */
9947
9948 static bool last_block;
9949
9950 /* True if we need a copy_state command at the start of the next block.  */
9951
9952 static bool need_copy_state;
9953
9954 #ifndef MAX_ARTIFICIAL_LABEL_BYTES
9955 # define MAX_ARTIFICIAL_LABEL_BYTES 30
9956 #endif
9957
9958 /* The function emits unwind directives for the start of an epilogue.  */
9959
9960 static void
9961 process_epilogue (FILE *asm_out_file, rtx insn ATTRIBUTE_UNUSED,
9962                   bool unwind, bool frame ATTRIBUTE_UNUSED)
9963 {
9964   /* If this isn't the last block of the function, then we need to label the
9965      current state, and copy it back in at the start of the next block.  */
9966
9967   if (!last_block)
9968     {
9969       if (unwind)
9970         fprintf (asm_out_file, "\t.label_state %d\n",
9971                  ++cfun->machine->state_num);
9972       need_copy_state = true;
9973     }
9974
9975   if (unwind)
9976     fprintf (asm_out_file, "\t.restore sp\n");
9977 }
9978
9979 /* This function processes a SET pattern for REG_CFA_ADJUST_CFA.  */
9980
9981 static void
9982 process_cfa_adjust_cfa (FILE *asm_out_file, rtx pat, rtx insn,
9983                         bool unwind, bool frame)
9984 {
9985   rtx dest = SET_DEST (pat);
9986   rtx src = SET_SRC (pat);
9987
9988   if (dest == stack_pointer_rtx)
9989     {
9990       if (GET_CODE (src) == PLUS)
9991         {
9992           rtx op0 = XEXP (src, 0);
9993           rtx op1 = XEXP (src, 1);
9994
9995           gcc_assert (op0 == dest && GET_CODE (op1) == CONST_INT);
9996
9997           if (INTVAL (op1) < 0)
9998             {
9999               gcc_assert (!frame_pointer_needed);
10000               if (unwind)
10001                 fprintf (asm_out_file,
10002                          "\t.fframe "HOST_WIDE_INT_PRINT_DEC"\n",
10003                          -INTVAL (op1));
10004             }
10005           else
10006             process_epilogue (asm_out_file, insn, unwind, frame);
10007         }
10008       else
10009         {
10010           gcc_assert (src == hard_frame_pointer_rtx);
10011           process_epilogue (asm_out_file, insn, unwind, frame);
10012         }
10013     }
10014   else if (dest == hard_frame_pointer_rtx)
10015     {
10016       gcc_assert (src == stack_pointer_rtx);
10017       gcc_assert (frame_pointer_needed);
10018
10019       if (unwind)
10020         fprintf (asm_out_file, "\t.vframe r%d\n",
10021                  ia64_dbx_register_number (REGNO (dest)));
10022     }
10023   else
10024     gcc_unreachable ();
10025 }
10026
10027 /* This function processes a SET pattern for REG_CFA_REGISTER.  */
10028
10029 static void
10030 process_cfa_register (FILE *asm_out_file, rtx pat, bool unwind)
10031 {
10032   rtx dest = SET_DEST (pat);
10033   rtx src = SET_SRC (pat);
10034   int dest_regno = REGNO (dest);
10035   int src_regno;
10036
10037   if (src == pc_rtx)
10038     {
10039       /* Saving return address pointer.  */
10040       if (unwind)
10041         fprintf (asm_out_file, "\t.save rp, r%d\n",
10042                  ia64_dbx_register_number (dest_regno));
10043       return;
10044     }
10045
10046   src_regno = REGNO (src);
10047
10048   switch (src_regno)
10049     {
10050     case PR_REG (0):
10051       gcc_assert (dest_regno == current_frame_info.r[reg_save_pr]);
10052       if (unwind)
10053         fprintf (asm_out_file, "\t.save pr, r%d\n",
10054                  ia64_dbx_register_number (dest_regno));
10055       break;
10056
10057     case AR_UNAT_REGNUM:
10058       gcc_assert (dest_regno == current_frame_info.r[reg_save_ar_unat]);
10059       if (unwind)
10060         fprintf (asm_out_file, "\t.save ar.unat, r%d\n",
10061                  ia64_dbx_register_number (dest_regno));
10062       break;
10063
10064     case AR_LC_REGNUM:
10065       gcc_assert (dest_regno == current_frame_info.r[reg_save_ar_lc]);
10066       if (unwind)
10067         fprintf (asm_out_file, "\t.save ar.lc, r%d\n",
10068                  ia64_dbx_register_number (dest_regno));
10069       break;
10070
10071     default:
10072       /* Everything else should indicate being stored to memory.  */
10073       gcc_unreachable ();
10074     }
10075 }
10076
10077 /* This function processes a SET pattern for REG_CFA_OFFSET.  */
10078
10079 static void
10080 process_cfa_offset (FILE *asm_out_file, rtx pat, bool unwind)
10081 {
10082   rtx dest = SET_DEST (pat);
10083   rtx src = SET_SRC (pat);
10084   int src_regno = REGNO (src);
10085   const char *saveop;
10086   HOST_WIDE_INT off;
10087   rtx base;
10088
10089   gcc_assert (MEM_P (dest));
10090   if (GET_CODE (XEXP (dest, 0)) == REG)
10091     {
10092       base = XEXP (dest, 0);
10093       off = 0;
10094     }
10095   else
10096     {
10097       gcc_assert (GET_CODE (XEXP (dest, 0)) == PLUS
10098                   && GET_CODE (XEXP (XEXP (dest, 0), 1)) == CONST_INT);
10099       base = XEXP (XEXP (dest, 0), 0);
10100       off = INTVAL (XEXP (XEXP (dest, 0), 1));
10101     }
10102
10103   if (base == hard_frame_pointer_rtx)
10104     {
10105       saveop = ".savepsp";
10106       off = - off;
10107     }
10108   else
10109     {
10110       gcc_assert (base == stack_pointer_rtx);
10111       saveop = ".savesp";
10112     }
10113
10114   src_regno = REGNO (src);
10115   switch (src_regno)
10116     {
10117     case BR_REG (0):
10118       gcc_assert (!current_frame_info.r[reg_save_b0]);
10119       if (unwind)
10120         fprintf (asm_out_file, "\t%s rp, " HOST_WIDE_INT_PRINT_DEC "\n",
10121                  saveop, off);
10122       break;
10123
10124     case PR_REG (0):
10125       gcc_assert (!current_frame_info.r[reg_save_pr]);
10126       if (unwind)
10127         fprintf (asm_out_file, "\t%s pr, " HOST_WIDE_INT_PRINT_DEC "\n",
10128                  saveop, off);
10129       break;
10130
10131     case AR_LC_REGNUM:
10132       gcc_assert (!current_frame_info.r[reg_save_ar_lc]);
10133       if (unwind)
10134         fprintf (asm_out_file, "\t%s ar.lc, " HOST_WIDE_INT_PRINT_DEC "\n",
10135                  saveop, off);
10136       break;
10137
10138     case AR_PFS_REGNUM:
10139       gcc_assert (!current_frame_info.r[reg_save_ar_pfs]);
10140       if (unwind)
10141         fprintf (asm_out_file, "\t%s ar.pfs, " HOST_WIDE_INT_PRINT_DEC "\n",
10142                  saveop, off);
10143       break;
10144
10145     case AR_UNAT_REGNUM:
10146       gcc_assert (!current_frame_info.r[reg_save_ar_unat]);
10147       if (unwind)
10148         fprintf (asm_out_file, "\t%s ar.unat, " HOST_WIDE_INT_PRINT_DEC "\n",
10149                  saveop, off);
10150       break;
10151
10152     case GR_REG (4):
10153     case GR_REG (5):
10154     case GR_REG (6):
10155     case GR_REG (7):
10156       if (unwind)
10157         fprintf (asm_out_file, "\t.save.g 0x%x\n",
10158                  1 << (src_regno - GR_REG (4)));
10159       break;
10160
10161     case BR_REG (1):
10162     case BR_REG (2):
10163     case BR_REG (3):
10164     case BR_REG (4):
10165     case BR_REG (5):
10166       if (unwind)
10167         fprintf (asm_out_file, "\t.save.b 0x%x\n",
10168                  1 << (src_regno - BR_REG (1)));
10169       break;
10170
10171     case FR_REG (2):
10172     case FR_REG (3):
10173     case FR_REG (4):
10174     case FR_REG (5):
10175       if (unwind)
10176         fprintf (asm_out_file, "\t.save.f 0x%x\n",
10177                  1 << (src_regno - FR_REG (2)));
10178       break;
10179
10180     case FR_REG (16): case FR_REG (17): case FR_REG (18): case FR_REG (19):
10181     case FR_REG (20): case FR_REG (21): case FR_REG (22): case FR_REG (23):
10182     case FR_REG (24): case FR_REG (25): case FR_REG (26): case FR_REG (27):
10183     case FR_REG (28): case FR_REG (29): case FR_REG (30): case FR_REG (31):
10184       if (unwind)
10185         fprintf (asm_out_file, "\t.save.gf 0x0, 0x%x\n",
10186                  1 << (src_regno - FR_REG (12)));
10187       break;
10188
10189     default:
10190       /* ??? For some reason we mark other general registers, even those
10191          we can't represent in the unwind info.  Ignore them.  */
10192       break;
10193     }
10194 }
10195
10196 /* This function looks at a single insn and emits any directives
10197    required to unwind this insn.  */
10198
10199 static void
10200 ia64_asm_unwind_emit (FILE *asm_out_file, rtx_insn *insn)
10201 {
10202   bool unwind = ia64_except_unwind_info (&global_options) == UI_TARGET;
10203   bool frame = dwarf2out_do_frame ();
10204   rtx note, pat;
10205   bool handled_one;
10206
10207   if (!unwind && !frame)
10208     return;
10209
10210   if (NOTE_INSN_BASIC_BLOCK_P (insn))
10211     {
10212       last_block = NOTE_BASIC_BLOCK (insn)->next_bb
10213      == EXIT_BLOCK_PTR_FOR_FN (cfun);
10214
10215       /* Restore unwind state from immediately before the epilogue.  */
10216       if (need_copy_state)
10217         {
10218           if (unwind)
10219             {
10220               fprintf (asm_out_file, "\t.body\n");
10221               fprintf (asm_out_file, "\t.copy_state %d\n",
10222                        cfun->machine->state_num);
10223             }
10224           need_copy_state = false;
10225         }
10226     }
10227
10228   if (NOTE_P (insn) || ! RTX_FRAME_RELATED_P (insn))
10229     return;
10230
10231   /* Look for the ALLOC insn.  */
10232   if (INSN_CODE (insn) == CODE_FOR_alloc)
10233     {
10234       rtx dest = SET_DEST (XVECEXP (PATTERN (insn), 0, 0));
10235       int dest_regno = REGNO (dest);
10236
10237       /* If this is the final destination for ar.pfs, then this must
10238          be the alloc in the prologue.  */
10239       if (dest_regno == current_frame_info.r[reg_save_ar_pfs])
10240         {
10241           if (unwind)
10242             fprintf (asm_out_file, "\t.save ar.pfs, r%d\n",
10243                      ia64_dbx_register_number (dest_regno));
10244         }
10245       else
10246         {
10247           /* This must be an alloc before a sibcall.  We must drop the
10248              old frame info.  The easiest way to drop the old frame
10249              info is to ensure we had a ".restore sp" directive
10250              followed by a new prologue.  If the procedure doesn't
10251              have a memory-stack frame, we'll issue a dummy ".restore
10252              sp" now.  */
10253           if (current_frame_info.total_size == 0 && !frame_pointer_needed)
10254             /* if haven't done process_epilogue() yet, do it now */
10255             process_epilogue (asm_out_file, insn, unwind, frame);
10256           if (unwind)
10257             fprintf (asm_out_file, "\t.prologue\n");
10258         }
10259       return;
10260     }
10261
10262   handled_one = false;
10263   for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
10264     switch (REG_NOTE_KIND (note))
10265       {
10266       case REG_CFA_ADJUST_CFA:
10267         pat = XEXP (note, 0);
10268         if (pat == NULL)
10269           pat = PATTERN (insn);
10270         process_cfa_adjust_cfa (asm_out_file, pat, insn, unwind, frame);
10271         handled_one = true;
10272         break;
10273
10274       case REG_CFA_OFFSET:
10275         pat = XEXP (note, 0);
10276         if (pat == NULL)
10277           pat = PATTERN (insn);
10278         process_cfa_offset (asm_out_file, pat, unwind);
10279         handled_one = true;
10280         break;
10281
10282       case REG_CFA_REGISTER:
10283         pat = XEXP (note, 0);
10284         if (pat == NULL)
10285           pat = PATTERN (insn);
10286         process_cfa_register (asm_out_file, pat, unwind);
10287         handled_one = true;
10288         break;
10289
10290       case REG_FRAME_RELATED_EXPR:
10291       case REG_CFA_DEF_CFA:
10292       case REG_CFA_EXPRESSION:
10293       case REG_CFA_RESTORE:
10294       case REG_CFA_SET_VDRAP:
10295         /* Not used in the ia64 port.  */
10296         gcc_unreachable ();
10297
10298       default:
10299         /* Not a frame-related note.  */
10300         break;
10301       }
10302
10303   /* All REG_FRAME_RELATED_P insns, besides ALLOC, are marked with the
10304      explicit action to take.  No guessing required.  */
10305   gcc_assert (handled_one);
10306 }
10307
10308 /* Implement TARGET_ASM_EMIT_EXCEPT_PERSONALITY.  */
10309
10310 static void
10311 ia64_asm_emit_except_personality (rtx personality)
10312 {
10313   fputs ("\t.personality\t", asm_out_file);
10314   output_addr_const (asm_out_file, personality);
10315   fputc ('\n', asm_out_file);
10316 }
10317
10318 /* Implement TARGET_ASM_INITIALIZE_SECTIONS.  */
10319
10320 static void
10321 ia64_asm_init_sections (void)
10322 {
10323   exception_section = get_unnamed_section (0, output_section_asm_op,
10324                                            "\t.handlerdata");
10325 }
10326
10327 /* Implement TARGET_DEBUG_UNWIND_INFO.  */
10328
10329 static enum unwind_info_type
10330 ia64_debug_unwind_info (void)
10331 {
10332   return UI_TARGET;
10333 }
10334 \f
10335 enum ia64_builtins
10336 {
10337   IA64_BUILTIN_BSP,
10338   IA64_BUILTIN_COPYSIGNQ,
10339   IA64_BUILTIN_FABSQ,
10340   IA64_BUILTIN_FLUSHRS,
10341   IA64_BUILTIN_INFQ,
10342   IA64_BUILTIN_HUGE_VALQ,
10343   IA64_BUILTIN_max
10344 };
10345
10346 static GTY(()) tree ia64_builtins[(int) IA64_BUILTIN_max];
10347
10348 void
10349 ia64_init_builtins (void)
10350 {
10351   tree fpreg_type;
10352   tree float80_type;
10353   tree decl;
10354
10355   /* The __fpreg type.  */
10356   fpreg_type = make_node (REAL_TYPE);
10357   TYPE_PRECISION (fpreg_type) = 82;
10358   layout_type (fpreg_type);
10359   (*lang_hooks.types.register_builtin_type) (fpreg_type, "__fpreg");
10360
10361   /* The __float80 type.  */
10362   float80_type = make_node (REAL_TYPE);
10363   TYPE_PRECISION (float80_type) = 80;
10364   layout_type (float80_type);
10365   (*lang_hooks.types.register_builtin_type) (float80_type, "__float80");
10366
10367   /* The __float128 type.  */
10368   if (!TARGET_HPUX)
10369     {
10370       tree ftype;
10371       tree float128_type = make_node (REAL_TYPE);
10372
10373       TYPE_PRECISION (float128_type) = 128;
10374       layout_type (float128_type);
10375       (*lang_hooks.types.register_builtin_type) (float128_type, "__float128");
10376
10377       /* TFmode support builtins.  */
10378       ftype = build_function_type_list (float128_type, NULL_TREE);
10379       decl = add_builtin_function ("__builtin_infq", ftype,
10380                                    IA64_BUILTIN_INFQ, BUILT_IN_MD,
10381                                    NULL, NULL_TREE);
10382       ia64_builtins[IA64_BUILTIN_INFQ] = decl;
10383
10384       decl = add_builtin_function ("__builtin_huge_valq", ftype,
10385                                    IA64_BUILTIN_HUGE_VALQ, BUILT_IN_MD,
10386                                    NULL, NULL_TREE);
10387       ia64_builtins[IA64_BUILTIN_HUGE_VALQ] = decl;
10388
10389       ftype = build_function_type_list (float128_type,
10390                                         float128_type,
10391                                         NULL_TREE);
10392       decl = add_builtin_function ("__builtin_fabsq", ftype,
10393                                    IA64_BUILTIN_FABSQ, BUILT_IN_MD,
10394                                    "__fabstf2", NULL_TREE);
10395       TREE_READONLY (decl) = 1;
10396       ia64_builtins[IA64_BUILTIN_FABSQ] = decl;
10397
10398       ftype = build_function_type_list (float128_type,
10399                                         float128_type,
10400                                         float128_type,
10401                                         NULL_TREE);
10402       decl = add_builtin_function ("__builtin_copysignq", ftype,
10403                                    IA64_BUILTIN_COPYSIGNQ, BUILT_IN_MD,
10404                                    "__copysigntf3", NULL_TREE);
10405       TREE_READONLY (decl) = 1;
10406       ia64_builtins[IA64_BUILTIN_COPYSIGNQ] = decl;
10407     }
10408   else
10409     /* Under HPUX, this is a synonym for "long double".  */
10410     (*lang_hooks.types.register_builtin_type) (long_double_type_node,
10411                                                "__float128");
10412
10413   /* Fwrite on VMS is non-standard.  */
10414 #if TARGET_ABI_OPEN_VMS
10415   vms_patch_builtins ();
10416 #endif
10417
10418 #define def_builtin(name, type, code)                                   \
10419   add_builtin_function ((name), (type), (code), BUILT_IN_MD,    \
10420                        NULL, NULL_TREE)
10421
10422   decl = def_builtin ("__builtin_ia64_bsp",
10423                       build_function_type_list (ptr_type_node, NULL_TREE),
10424                       IA64_BUILTIN_BSP);
10425   ia64_builtins[IA64_BUILTIN_BSP] = decl;
10426
10427   decl = def_builtin ("__builtin_ia64_flushrs",
10428                       build_function_type_list (void_type_node, NULL_TREE),
10429                       IA64_BUILTIN_FLUSHRS);
10430   ia64_builtins[IA64_BUILTIN_FLUSHRS] = decl;
10431
10432 #undef def_builtin
10433
10434   if (TARGET_HPUX)
10435     {
10436       if ((decl = builtin_decl_explicit (BUILT_IN_FINITE)) != NULL_TREE)
10437         set_user_assembler_name (decl, "_Isfinite");
10438       if ((decl = builtin_decl_explicit (BUILT_IN_FINITEF)) != NULL_TREE)
10439         set_user_assembler_name (decl, "_Isfinitef");
10440       if ((decl = builtin_decl_explicit (BUILT_IN_FINITEL)) != NULL_TREE)
10441         set_user_assembler_name (decl, "_Isfinitef128");
10442     }
10443 }
10444
10445 rtx
10446 ia64_expand_builtin (tree exp, rtx target, rtx subtarget ATTRIBUTE_UNUSED,
10447                      machine_mode mode ATTRIBUTE_UNUSED,
10448                      int ignore ATTRIBUTE_UNUSED)
10449 {
10450   tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
10451   unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
10452
10453   switch (fcode)
10454     {
10455     case IA64_BUILTIN_BSP:
10456       if (! target || ! register_operand (target, DImode))
10457         target = gen_reg_rtx (DImode);
10458       emit_insn (gen_bsp_value (target));
10459 #ifdef POINTERS_EXTEND_UNSIGNED
10460       target = convert_memory_address (ptr_mode, target);
10461 #endif
10462       return target;
10463
10464     case IA64_BUILTIN_FLUSHRS:
10465       emit_insn (gen_flushrs ());
10466       return const0_rtx;
10467
10468     case IA64_BUILTIN_INFQ:
10469     case IA64_BUILTIN_HUGE_VALQ:
10470       {
10471         machine_mode target_mode = TYPE_MODE (TREE_TYPE (exp));
10472         REAL_VALUE_TYPE inf;
10473         rtx tmp;
10474
10475         real_inf (&inf);
10476         tmp = CONST_DOUBLE_FROM_REAL_VALUE (inf, target_mode);
10477
10478         tmp = validize_mem (force_const_mem (target_mode, tmp));
10479
10480         if (target == 0)
10481           target = gen_reg_rtx (target_mode);
10482
10483         emit_move_insn (target, tmp);
10484         return target;
10485       }
10486
10487     case IA64_BUILTIN_FABSQ:
10488     case IA64_BUILTIN_COPYSIGNQ:
10489       return expand_call (exp, target, ignore);
10490
10491     default:
10492       gcc_unreachable ();
10493     }
10494
10495   return NULL_RTX;
10496 }
10497
10498 /* Return the ia64 builtin for CODE.  */
10499
10500 static tree
10501 ia64_builtin_decl (unsigned code, bool initialize_p ATTRIBUTE_UNUSED)
10502 {
10503   if (code >= IA64_BUILTIN_max)
10504     return error_mark_node;
10505
10506   return ia64_builtins[code];
10507 }
10508
10509 /* For the HP-UX IA64 aggregate parameters are passed stored in the
10510    most significant bits of the stack slot.  */
10511
10512 enum direction
10513 ia64_hpux_function_arg_padding (machine_mode mode, const_tree type)
10514 {
10515    /* Exception to normal case for structures/unions/etc.  */
10516
10517    if (type && AGGREGATE_TYPE_P (type)
10518        && int_size_in_bytes (type) < UNITS_PER_WORD)
10519      return upward;
10520
10521    /* Fall back to the default.  */
10522    return DEFAULT_FUNCTION_ARG_PADDING (mode, type);
10523 }
10524
10525 /* Emit text to declare externally defined variables and functions, because
10526    the Intel assembler does not support undefined externals.  */
10527
10528 void
10529 ia64_asm_output_external (FILE *file, tree decl, const char *name)
10530 {
10531   /* We output the name if and only if TREE_SYMBOL_REFERENCED is
10532      set in order to avoid putting out names that are never really
10533      used. */
10534   if (TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (decl)))
10535     {
10536       /* maybe_assemble_visibility will return 1 if the assembler
10537          visibility directive is output.  */
10538       int need_visibility = ((*targetm.binds_local_p) (decl)
10539                              && maybe_assemble_visibility (decl));
10540
10541       /* GNU as does not need anything here, but the HP linker does
10542          need something for external functions.  */
10543       if ((TARGET_HPUX_LD || !TARGET_GNU_AS)
10544           && TREE_CODE (decl) == FUNCTION_DECL)
10545           (*targetm.asm_out.globalize_decl_name) (file, decl);
10546       else if (need_visibility && !TARGET_GNU_AS)
10547         (*targetm.asm_out.globalize_label) (file, name);
10548     }
10549 }
10550
10551 /* Set SImode div/mod functions, init_integral_libfuncs only initializes
10552    modes of word_mode and larger.  Rename the TFmode libfuncs using the
10553    HPUX conventions. __divtf3 is used for XFmode. We need to keep it for
10554    backward compatibility. */
10555
10556 static void
10557 ia64_init_libfuncs (void)
10558 {
10559   set_optab_libfunc (sdiv_optab, SImode, "__divsi3");
10560   set_optab_libfunc (udiv_optab, SImode, "__udivsi3");
10561   set_optab_libfunc (smod_optab, SImode, "__modsi3");
10562   set_optab_libfunc (umod_optab, SImode, "__umodsi3");
10563
10564   set_optab_libfunc (add_optab, TFmode, "_U_Qfadd");
10565   set_optab_libfunc (sub_optab, TFmode, "_U_Qfsub");
10566   set_optab_libfunc (smul_optab, TFmode, "_U_Qfmpy");
10567   set_optab_libfunc (sdiv_optab, TFmode, "_U_Qfdiv");
10568   set_optab_libfunc (neg_optab, TFmode, "_U_Qfneg");
10569
10570   set_conv_libfunc (sext_optab, TFmode, SFmode, "_U_Qfcnvff_sgl_to_quad");
10571   set_conv_libfunc (sext_optab, TFmode, DFmode, "_U_Qfcnvff_dbl_to_quad");
10572   set_conv_libfunc (sext_optab, TFmode, XFmode, "_U_Qfcnvff_f80_to_quad");
10573   set_conv_libfunc (trunc_optab, SFmode, TFmode, "_U_Qfcnvff_quad_to_sgl");
10574   set_conv_libfunc (trunc_optab, DFmode, TFmode, "_U_Qfcnvff_quad_to_dbl");
10575   set_conv_libfunc (trunc_optab, XFmode, TFmode, "_U_Qfcnvff_quad_to_f80");
10576
10577   set_conv_libfunc (sfix_optab, SImode, TFmode, "_U_Qfcnvfxt_quad_to_sgl");
10578   set_conv_libfunc (sfix_optab, DImode, TFmode, "_U_Qfcnvfxt_quad_to_dbl");
10579   set_conv_libfunc (sfix_optab, TImode, TFmode, "_U_Qfcnvfxt_quad_to_quad");
10580   set_conv_libfunc (ufix_optab, SImode, TFmode, "_U_Qfcnvfxut_quad_to_sgl");
10581   set_conv_libfunc (ufix_optab, DImode, TFmode, "_U_Qfcnvfxut_quad_to_dbl");
10582
10583   set_conv_libfunc (sfloat_optab, TFmode, SImode, "_U_Qfcnvxf_sgl_to_quad");
10584   set_conv_libfunc (sfloat_optab, TFmode, DImode, "_U_Qfcnvxf_dbl_to_quad");
10585   set_conv_libfunc (sfloat_optab, TFmode, TImode, "_U_Qfcnvxf_quad_to_quad");
10586   /* HP-UX 11.23 libc does not have a function for unsigned
10587      SImode-to-TFmode conversion.  */
10588   set_conv_libfunc (ufloat_optab, TFmode, DImode, "_U_Qfcnvxuf_dbl_to_quad");
10589 }
10590
10591 /* Rename all the TFmode libfuncs using the HPUX conventions.  */
10592
10593 static void
10594 ia64_hpux_init_libfuncs (void)
10595 {
10596   ia64_init_libfuncs ();
10597
10598   /* The HP SI millicode division and mod functions expect DI arguments.
10599      By turning them off completely we avoid using both libgcc and the
10600      non-standard millicode routines and use the HP DI millicode routines
10601      instead.  */
10602
10603   set_optab_libfunc (sdiv_optab, SImode, 0);
10604   set_optab_libfunc (udiv_optab, SImode, 0);
10605   set_optab_libfunc (smod_optab, SImode, 0);
10606   set_optab_libfunc (umod_optab, SImode, 0);
10607
10608   set_optab_libfunc (sdiv_optab, DImode, "__milli_divI");
10609   set_optab_libfunc (udiv_optab, DImode, "__milli_divU");
10610   set_optab_libfunc (smod_optab, DImode, "__milli_remI");
10611   set_optab_libfunc (umod_optab, DImode, "__milli_remU");
10612
10613   /* HP-UX libc has TF min/max/abs routines in it.  */
10614   set_optab_libfunc (smin_optab, TFmode, "_U_Qfmin");
10615   set_optab_libfunc (smax_optab, TFmode, "_U_Qfmax");
10616   set_optab_libfunc (abs_optab, TFmode, "_U_Qfabs");
10617
10618   /* ia64_expand_compare uses this.  */
10619   cmptf_libfunc = init_one_libfunc ("_U_Qfcmp");
10620
10621   /* These should never be used.  */
10622   set_optab_libfunc (eq_optab, TFmode, 0);
10623   set_optab_libfunc (ne_optab, TFmode, 0);
10624   set_optab_libfunc (gt_optab, TFmode, 0);
10625   set_optab_libfunc (ge_optab, TFmode, 0);
10626   set_optab_libfunc (lt_optab, TFmode, 0);
10627   set_optab_libfunc (le_optab, TFmode, 0);
10628 }
10629
10630 /* Rename the division and modulus functions in VMS.  */
10631
10632 static void
10633 ia64_vms_init_libfuncs (void)
10634 {
10635   set_optab_libfunc (sdiv_optab, SImode, "OTS$DIV_I");
10636   set_optab_libfunc (sdiv_optab, DImode, "OTS$DIV_L");
10637   set_optab_libfunc (udiv_optab, SImode, "OTS$DIV_UI");
10638   set_optab_libfunc (udiv_optab, DImode, "OTS$DIV_UL");
10639   set_optab_libfunc (smod_optab, SImode, "OTS$REM_I");
10640   set_optab_libfunc (smod_optab, DImode, "OTS$REM_L");
10641   set_optab_libfunc (umod_optab, SImode, "OTS$REM_UI");
10642   set_optab_libfunc (umod_optab, DImode, "OTS$REM_UL");
10643   abort_libfunc = init_one_libfunc ("decc$abort");
10644   memcmp_libfunc = init_one_libfunc ("decc$memcmp");
10645 #ifdef MEM_LIBFUNCS_INIT
10646   MEM_LIBFUNCS_INIT;
10647 #endif
10648 }
10649
10650 /* Rename the TFmode libfuncs available from soft-fp in glibc using
10651    the HPUX conventions.  */
10652
10653 static void
10654 ia64_sysv4_init_libfuncs (void)
10655 {
10656   ia64_init_libfuncs ();
10657
10658   /* These functions are not part of the HPUX TFmode interface.  We
10659      use them instead of _U_Qfcmp, which doesn't work the way we
10660      expect.  */
10661   set_optab_libfunc (eq_optab, TFmode, "_U_Qfeq");
10662   set_optab_libfunc (ne_optab, TFmode, "_U_Qfne");
10663   set_optab_libfunc (gt_optab, TFmode, "_U_Qfgt");
10664   set_optab_libfunc (ge_optab, TFmode, "_U_Qfge");
10665   set_optab_libfunc (lt_optab, TFmode, "_U_Qflt");
10666   set_optab_libfunc (le_optab, TFmode, "_U_Qfle");
10667
10668   /* We leave out _U_Qfmin, _U_Qfmax and _U_Qfabs since soft-fp in
10669      glibc doesn't have them.  */
10670 }
10671
10672 /* Use soft-fp.  */
10673
10674 static void
10675 ia64_soft_fp_init_libfuncs (void)
10676 {
10677 }
10678
10679 static bool
10680 ia64_vms_valid_pointer_mode (machine_mode mode)
10681 {
10682   return (mode == SImode || mode == DImode);
10683 }
10684 \f
10685 /* For HPUX, it is illegal to have relocations in shared segments.  */
10686
10687 static int
10688 ia64_hpux_reloc_rw_mask (void)
10689 {
10690   return 3;
10691 }
10692
10693 /* For others, relax this so that relocations to local data goes in
10694    read-only segments, but we still cannot allow global relocations
10695    in read-only segments.  */
10696
10697 static int
10698 ia64_reloc_rw_mask (void)
10699 {
10700   return flag_pic ? 3 : 2;
10701 }
10702
10703 /* Return the section to use for X.  The only special thing we do here
10704    is to honor small data.  */
10705
10706 static section *
10707 ia64_select_rtx_section (machine_mode mode, rtx x,
10708                          unsigned HOST_WIDE_INT align)
10709 {
10710   if (GET_MODE_SIZE (mode) > 0
10711       && GET_MODE_SIZE (mode) <= ia64_section_threshold
10712       && !TARGET_NO_SDATA)
10713     return sdata_section;
10714   else
10715     return default_elf_select_rtx_section (mode, x, align);
10716 }
10717
10718 static unsigned int
10719 ia64_section_type_flags (tree decl, const char *name, int reloc)
10720 {
10721   unsigned int flags = 0;
10722
10723   if (strcmp (name, ".sdata") == 0
10724       || strncmp (name, ".sdata.", 7) == 0
10725       || strncmp (name, ".gnu.linkonce.s.", 16) == 0
10726       || strncmp (name, ".sdata2.", 8) == 0
10727       || strncmp (name, ".gnu.linkonce.s2.", 17) == 0
10728       || strcmp (name, ".sbss") == 0
10729       || strncmp (name, ".sbss.", 6) == 0
10730       || strncmp (name, ".gnu.linkonce.sb.", 17) == 0)
10731     flags = SECTION_SMALL;
10732
10733   flags |= default_section_type_flags (decl, name, reloc);
10734   return flags;
10735 }
10736
10737 /* Returns true if FNTYPE (a FUNCTION_TYPE or a METHOD_TYPE) returns a
10738    structure type and that the address of that type should be passed
10739    in out0, rather than in r8.  */
10740
10741 static bool
10742 ia64_struct_retval_addr_is_first_parm_p (tree fntype)
10743 {
10744   tree ret_type = TREE_TYPE (fntype);
10745
10746   /* The Itanium C++ ABI requires that out0, rather than r8, be used
10747      as the structure return address parameter, if the return value
10748      type has a non-trivial copy constructor or destructor.  It is not
10749      clear if this same convention should be used for other
10750      programming languages.  Until G++ 3.4, we incorrectly used r8 for
10751      these return values.  */
10752   return (abi_version_at_least (2)
10753           && ret_type
10754           && TYPE_MODE (ret_type) == BLKmode
10755           && TREE_ADDRESSABLE (ret_type)
10756           && strcmp (lang_hooks.name, "GNU C++") == 0);
10757 }
10758
10759 /* Output the assembler code for a thunk function.  THUNK_DECL is the
10760    declaration for the thunk function itself, FUNCTION is the decl for
10761    the target function.  DELTA is an immediate constant offset to be
10762    added to THIS.  If VCALL_OFFSET is nonzero, the word at
10763    *(*this + vcall_offset) should be added to THIS.  */
10764
10765 static void
10766 ia64_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED,
10767                       HOST_WIDE_INT delta, HOST_WIDE_INT vcall_offset,
10768                       tree function)
10769 {
10770   rtx this_rtx, funexp;
10771   rtx_insn *insn;
10772   unsigned int this_parmno;
10773   unsigned int this_regno;
10774   rtx delta_rtx;
10775
10776   reload_completed = 1;
10777   epilogue_completed = 1;
10778
10779   /* Set things up as ia64_expand_prologue might.  */
10780   last_scratch_gr_reg = 15;
10781
10782   memset (&current_frame_info, 0, sizeof (current_frame_info));
10783   current_frame_info.spill_cfa_off = -16;
10784   current_frame_info.n_input_regs = 1;
10785   current_frame_info.need_regstk = (TARGET_REG_NAMES != 0);
10786
10787   /* Mark the end of the (empty) prologue.  */
10788   emit_note (NOTE_INSN_PROLOGUE_END);
10789
10790   /* Figure out whether "this" will be the first parameter (the
10791      typical case) or the second parameter (as happens when the
10792      virtual function returns certain class objects).  */
10793   this_parmno
10794     = (ia64_struct_retval_addr_is_first_parm_p (TREE_TYPE (thunk))
10795        ? 1 : 0);
10796   this_regno = IN_REG (this_parmno);
10797   if (!TARGET_REG_NAMES)
10798     reg_names[this_regno] = ia64_reg_numbers[this_parmno];
10799
10800   this_rtx = gen_rtx_REG (Pmode, this_regno);
10801
10802   /* Apply the constant offset, if required.  */
10803   delta_rtx = GEN_INT (delta);
10804   if (TARGET_ILP32)
10805     {
10806       rtx tmp = gen_rtx_REG (ptr_mode, this_regno);
10807       REG_POINTER (tmp) = 1;
10808       if (delta && satisfies_constraint_I (delta_rtx))
10809         {
10810           emit_insn (gen_ptr_extend_plus_imm (this_rtx, tmp, delta_rtx));
10811           delta = 0;
10812         }
10813       else
10814         emit_insn (gen_ptr_extend (this_rtx, tmp));
10815     }
10816   if (delta)
10817     {
10818       if (!satisfies_constraint_I (delta_rtx))
10819         {
10820           rtx tmp = gen_rtx_REG (Pmode, 2);
10821           emit_move_insn (tmp, delta_rtx);
10822           delta_rtx = tmp;
10823         }
10824       emit_insn (gen_adddi3 (this_rtx, this_rtx, delta_rtx));
10825     }
10826
10827   /* Apply the offset from the vtable, if required.  */
10828   if (vcall_offset)
10829     {
10830       rtx vcall_offset_rtx = GEN_INT (vcall_offset);
10831       rtx tmp = gen_rtx_REG (Pmode, 2);
10832
10833       if (TARGET_ILP32)
10834         {
10835           rtx t = gen_rtx_REG (ptr_mode, 2);
10836           REG_POINTER (t) = 1;
10837           emit_move_insn (t, gen_rtx_MEM (ptr_mode, this_rtx));
10838           if (satisfies_constraint_I (vcall_offset_rtx))
10839             {
10840               emit_insn (gen_ptr_extend_plus_imm (tmp, t, vcall_offset_rtx));
10841               vcall_offset = 0;
10842             }
10843           else
10844             emit_insn (gen_ptr_extend (tmp, t));
10845         }
10846       else
10847         emit_move_insn (tmp, gen_rtx_MEM (Pmode, this_rtx));
10848
10849       if (vcall_offset)
10850         {
10851           if (!satisfies_constraint_J (vcall_offset_rtx))
10852             {
10853               rtx tmp2 = gen_rtx_REG (Pmode, next_scratch_gr_reg ());
10854               emit_move_insn (tmp2, vcall_offset_rtx);
10855               vcall_offset_rtx = tmp2;
10856             }
10857           emit_insn (gen_adddi3 (tmp, tmp, vcall_offset_rtx));
10858         }
10859
10860       if (TARGET_ILP32)
10861         emit_insn (gen_zero_extendsidi2 (tmp, gen_rtx_MEM (ptr_mode, tmp)));
10862       else
10863         emit_move_insn (tmp, gen_rtx_MEM (Pmode, tmp));
10864
10865       emit_insn (gen_adddi3 (this_rtx, this_rtx, tmp));
10866     }
10867
10868   /* Generate a tail call to the target function.  */
10869   if (! TREE_USED (function))
10870     {
10871       assemble_external (function);
10872       TREE_USED (function) = 1;
10873     }
10874   funexp = XEXP (DECL_RTL (function), 0);
10875   funexp = gen_rtx_MEM (FUNCTION_MODE, funexp);
10876   ia64_expand_call (NULL_RTX, funexp, NULL_RTX, 1);
10877   insn = get_last_insn ();
10878   SIBLING_CALL_P (insn) = 1;
10879
10880   /* Code generation for calls relies on splitting.  */
10881   reload_completed = 1;
10882   epilogue_completed = 1;
10883   try_split (PATTERN (insn), insn, 0);
10884
10885   emit_barrier ();
10886
10887   /* Run just enough of rest_of_compilation to get the insns emitted.
10888      There's not really enough bulk here to make other passes such as
10889      instruction scheduling worth while.  Note that use_thunk calls
10890      assemble_start_function and assemble_end_function.  */
10891
10892   emit_all_insn_group_barriers (NULL);
10893   insn = get_insns ();
10894   shorten_branches (insn);
10895   final_start_function (insn, file, 1);
10896   final (insn, file, 1);
10897   final_end_function ();
10898
10899   reload_completed = 0;
10900   epilogue_completed = 0;
10901 }
10902
10903 /* Worker function for TARGET_STRUCT_VALUE_RTX.  */
10904
10905 static rtx
10906 ia64_struct_value_rtx (tree fntype,
10907                        int incoming ATTRIBUTE_UNUSED)
10908 {
10909   if (TARGET_ABI_OPEN_VMS ||
10910       (fntype && ia64_struct_retval_addr_is_first_parm_p (fntype)))
10911     return NULL_RTX;
10912   return gen_rtx_REG (Pmode, GR_REG (8));
10913 }
10914
10915 static bool
10916 ia64_scalar_mode_supported_p (machine_mode mode)
10917 {
10918   switch (mode)
10919     {
10920     case QImode:
10921     case HImode:
10922     case SImode:
10923     case DImode:
10924     case TImode:
10925       return true;
10926
10927     case SFmode:
10928     case DFmode:
10929     case XFmode:
10930     case RFmode:
10931       return true;
10932
10933     case TFmode:
10934       return true;
10935
10936     default:
10937       return false;
10938     }
10939 }
10940
10941 static bool
10942 ia64_vector_mode_supported_p (machine_mode mode)
10943 {
10944   switch (mode)
10945     {
10946     case V8QImode:
10947     case V4HImode:
10948     case V2SImode:
10949       return true;
10950
10951     case V2SFmode:
10952       return true;
10953
10954     default:
10955       return false;
10956     }
10957 }
10958
10959 /* Implement TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P.  */
10960
10961 static bool
10962 ia64_libgcc_floating_mode_supported_p (machine_mode mode)
10963 {
10964   switch (mode)
10965     {
10966     case SFmode:
10967     case DFmode:
10968       return true;
10969
10970     case XFmode:
10971 #ifdef IA64_NO_LIBGCC_XFMODE
10972       return false;
10973 #else
10974       return true;
10975 #endif
10976
10977     case TFmode:
10978 #ifdef IA64_NO_LIBGCC_TFMODE
10979       return false;
10980 #else
10981       return true;
10982 #endif
10983
10984     default:
10985       return false;
10986     }
10987 }
10988
10989 /* Implement the FUNCTION_PROFILER macro.  */
10990
10991 void
10992 ia64_output_function_profiler (FILE *file, int labelno)
10993 {
10994   bool indirect_call;
10995
10996   /* If the function needs a static chain and the static chain
10997      register is r15, we use an indirect call so as to bypass
10998      the PLT stub in case the executable is dynamically linked,
10999      because the stub clobbers r15 as per 5.3.6 of the psABI.
11000      We don't need to do that in non canonical PIC mode.  */
11001
11002   if (cfun->static_chain_decl && !TARGET_NO_PIC && !TARGET_AUTO_PIC)
11003     {
11004       gcc_assert (STATIC_CHAIN_REGNUM == 15);
11005       indirect_call = true;
11006     }
11007   else
11008     indirect_call = false;
11009
11010   if (TARGET_GNU_AS)
11011     fputs ("\t.prologue 4, r40\n", file);
11012   else
11013     fputs ("\t.prologue\n\t.save ar.pfs, r40\n", file);
11014   fputs ("\talloc out0 = ar.pfs, 8, 0, 4, 0\n", file);
11015
11016   if (NO_PROFILE_COUNTERS)
11017     fputs ("\tmov out3 = r0\n", file);
11018   else
11019     {
11020       char buf[20];
11021       ASM_GENERATE_INTERNAL_LABEL (buf, "LP", labelno);
11022
11023       if (TARGET_AUTO_PIC)
11024         fputs ("\tmovl out3 = @gprel(", file);
11025       else
11026         fputs ("\taddl out3 = @ltoff(", file);
11027       assemble_name (file, buf);
11028       if (TARGET_AUTO_PIC)
11029         fputs (")\n", file);
11030       else
11031         fputs ("), r1\n", file);
11032     }
11033
11034   if (indirect_call)
11035     fputs ("\taddl r14 = @ltoff(@fptr(_mcount)), r1\n", file);
11036   fputs ("\t;;\n", file);
11037
11038   fputs ("\t.save rp, r42\n", file);
11039   fputs ("\tmov out2 = b0\n", file);
11040   if (indirect_call)
11041     fputs ("\tld8 r14 = [r14]\n\t;;\n", file);
11042   fputs ("\t.body\n", file);
11043   fputs ("\tmov out1 = r1\n", file);
11044   if (indirect_call)
11045     {
11046       fputs ("\tld8 r16 = [r14], 8\n\t;;\n", file);
11047       fputs ("\tmov b6 = r16\n", file);
11048       fputs ("\tld8 r1 = [r14]\n", file);
11049       fputs ("\tbr.call.sptk.many b0 = b6\n\t;;\n", file);
11050     }
11051   else
11052     fputs ("\tbr.call.sptk.many b0 = _mcount\n\t;;\n", file);
11053 }
11054
11055 static GTY(()) rtx mcount_func_rtx;
11056 static rtx
11057 gen_mcount_func_rtx (void)
11058 {
11059   if (!mcount_func_rtx)
11060     mcount_func_rtx = init_one_libfunc ("_mcount");
11061   return mcount_func_rtx;
11062 }
11063
11064 void
11065 ia64_profile_hook (int labelno)
11066 {
11067   rtx label, ip;
11068
11069   if (NO_PROFILE_COUNTERS)
11070     label = const0_rtx;
11071   else
11072     {
11073       char buf[30];
11074       const char *label_name;
11075       ASM_GENERATE_INTERNAL_LABEL (buf, "LP", labelno);
11076       label_name = ggc_strdup ((*targetm.strip_name_encoding) (buf));
11077       label = gen_rtx_SYMBOL_REF (Pmode, label_name);
11078       SYMBOL_REF_FLAGS (label) = SYMBOL_FLAG_LOCAL;
11079     }
11080   ip = gen_reg_rtx (Pmode);
11081   emit_insn (gen_ip_value (ip));
11082   emit_library_call (gen_mcount_func_rtx (), LCT_NORMAL,
11083                      VOIDmode, 3,
11084                      gen_rtx_REG (Pmode, BR_REG (0)), Pmode,
11085                      ip, Pmode,
11086                      label, Pmode);
11087 }
11088
11089 /* Return the mangling of TYPE if it is an extended fundamental type.  */
11090
11091 static const char *
11092 ia64_mangle_type (const_tree type)
11093 {
11094   type = TYPE_MAIN_VARIANT (type);
11095
11096   if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
11097       && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
11098     return NULL;
11099
11100   /* On HP-UX, "long double" is mangled as "e" so __float128 is
11101      mangled as "e".  */
11102   if (!TARGET_HPUX && TYPE_MODE (type) == TFmode)
11103     return "g";
11104   /* On HP-UX, "e" is not available as a mangling of __float80 so use
11105      an extended mangling.  Elsewhere, "e" is available since long
11106      double is 80 bits.  */
11107   if (TYPE_MODE (type) == XFmode)
11108     return TARGET_HPUX ? "u9__float80" : "e";
11109   if (TYPE_MODE (type) == RFmode)
11110     return "u7__fpreg";
11111   return NULL;
11112 }
11113
11114 /* Return the diagnostic message string if conversion from FROMTYPE to
11115    TOTYPE is not allowed, NULL otherwise.  */
11116 static const char *
11117 ia64_invalid_conversion (const_tree fromtype, const_tree totype)
11118 {
11119   /* Reject nontrivial conversion to or from __fpreg.  */
11120   if (TYPE_MODE (fromtype) == RFmode
11121       && TYPE_MODE (totype) != RFmode
11122       && TYPE_MODE (totype) != VOIDmode)
11123     return N_("invalid conversion from %<__fpreg%>");
11124   if (TYPE_MODE (totype) == RFmode
11125       && TYPE_MODE (fromtype) != RFmode)
11126     return N_("invalid conversion to %<__fpreg%>");
11127   return NULL;
11128 }
11129
11130 /* Return the diagnostic message string if the unary operation OP is
11131    not permitted on TYPE, NULL otherwise.  */
11132 static const char *
11133 ia64_invalid_unary_op (int op, const_tree type)
11134 {
11135   /* Reject operations on __fpreg other than unary + or &.  */
11136   if (TYPE_MODE (type) == RFmode
11137       && op != CONVERT_EXPR
11138       && op != ADDR_EXPR)
11139     return N_("invalid operation on %<__fpreg%>");
11140   return NULL;
11141 }
11142
11143 /* Return the diagnostic message string if the binary operation OP is
11144    not permitted on TYPE1 and TYPE2, NULL otherwise.  */
11145 static const char *
11146 ia64_invalid_binary_op (int op ATTRIBUTE_UNUSED, const_tree type1, const_tree type2)
11147 {
11148   /* Reject operations on __fpreg.  */
11149   if (TYPE_MODE (type1) == RFmode || TYPE_MODE (type2) == RFmode)
11150     return N_("invalid operation on %<__fpreg%>");
11151   return NULL;
11152 }
11153
11154 /* HP-UX version_id attribute.
11155    For object foo, if the version_id is set to 1234 put out an alias
11156    of '.alias foo "foo{1234}"  We can't use "foo{1234}" in anything
11157    other than an alias statement because it is an illegal symbol name.  */
11158
11159 static tree
11160 ia64_handle_version_id_attribute (tree *node ATTRIBUTE_UNUSED,
11161                                  tree name ATTRIBUTE_UNUSED,
11162                                  tree args,
11163                                  int flags ATTRIBUTE_UNUSED,
11164                                  bool *no_add_attrs)
11165 {
11166   tree arg = TREE_VALUE (args);
11167
11168   if (TREE_CODE (arg) != STRING_CST)
11169     {
11170       error("version attribute is not a string");
11171       *no_add_attrs = true;
11172       return NULL_TREE;
11173     }
11174   return NULL_TREE;
11175 }
11176
11177 /* Target hook for c_mode_for_suffix.  */
11178
11179 static machine_mode
11180 ia64_c_mode_for_suffix (char suffix)
11181 {
11182   if (suffix == 'q')
11183     return TFmode;
11184   if (suffix == 'w')
11185     return XFmode;
11186
11187   return VOIDmode;
11188 }
11189
11190 static GTY(()) rtx ia64_dconst_0_5_rtx;
11191
11192 rtx
11193 ia64_dconst_0_5 (void)
11194 {
11195   if (! ia64_dconst_0_5_rtx)
11196     {
11197       REAL_VALUE_TYPE rv;
11198       real_from_string (&rv, "0.5");
11199       ia64_dconst_0_5_rtx = const_double_from_real_value (rv, DFmode);
11200     }
11201   return ia64_dconst_0_5_rtx;
11202 }
11203
11204 static GTY(()) rtx ia64_dconst_0_375_rtx;
11205
11206 rtx
11207 ia64_dconst_0_375 (void)
11208 {
11209   if (! ia64_dconst_0_375_rtx)
11210     {
11211       REAL_VALUE_TYPE rv;
11212       real_from_string (&rv, "0.375");
11213       ia64_dconst_0_375_rtx = const_double_from_real_value (rv, DFmode);
11214     }
11215   return ia64_dconst_0_375_rtx;
11216 }
11217
11218 static machine_mode
11219 ia64_get_reg_raw_mode (int regno)
11220 {
11221   if (FR_REGNO_P (regno))
11222     return XFmode;
11223   return default_get_reg_raw_mode(regno);
11224 }
11225
11226 /* Implement TARGET_MEMBER_TYPE_FORCES_BLK.  ??? Might not be needed
11227    anymore.  */
11228
11229 bool
11230 ia64_member_type_forces_blk (const_tree, machine_mode mode)
11231 {
11232   return TARGET_HPUX && mode == TFmode;
11233 }
11234
11235 /* Always default to .text section until HP-UX linker is fixed.  */
11236
11237 ATTRIBUTE_UNUSED static section *
11238 ia64_hpux_function_section (tree decl ATTRIBUTE_UNUSED,
11239                             enum node_frequency freq ATTRIBUTE_UNUSED,
11240                             bool startup ATTRIBUTE_UNUSED,
11241                             bool exit ATTRIBUTE_UNUSED)
11242 {
11243   return NULL;
11244 }
11245 \f
11246 /* Construct (set target (vec_select op0 (parallel perm))) and
11247    return true if that's a valid instruction in the active ISA.  */
11248
11249 static bool
11250 expand_vselect (rtx target, rtx op0, const unsigned char *perm, unsigned nelt)
11251 {
11252   rtx rperm[MAX_VECT_LEN], x;
11253   unsigned i;
11254
11255   for (i = 0; i < nelt; ++i)
11256     rperm[i] = GEN_INT (perm[i]);
11257
11258   x = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelt, rperm));
11259   x = gen_rtx_VEC_SELECT (GET_MODE (target), op0, x);
11260   x = gen_rtx_SET (VOIDmode, target, x);
11261
11262   rtx_insn *insn = emit_insn (x);
11263   if (recog_memoized (insn) < 0)
11264     {
11265       remove_insn (insn);
11266       return false;
11267     }
11268   return true;
11269 }
11270
11271 /* Similar, but generate a vec_concat from op0 and op1 as well.  */
11272
11273 static bool
11274 expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
11275                         const unsigned char *perm, unsigned nelt)
11276 {
11277   machine_mode v2mode;
11278   rtx x;
11279
11280   v2mode = GET_MODE_2XWIDER_MODE (GET_MODE (op0));
11281   x = gen_rtx_VEC_CONCAT (v2mode, op0, op1);
11282   return expand_vselect (target, x, perm, nelt);
11283 }
11284
11285 /* Try to expand a no-op permutation.  */
11286
11287 static bool
11288 expand_vec_perm_identity (struct expand_vec_perm_d *d)
11289 {
11290   unsigned i, nelt = d->nelt;
11291
11292   for (i = 0; i < nelt; ++i)
11293     if (d->perm[i] != i)
11294       return false;
11295
11296   if (!d->testing_p)
11297     emit_move_insn (d->target, d->op0);
11298
11299   return true;
11300 }
11301
11302 /* Try to expand D via a shrp instruction.  */
11303
11304 static bool
11305 expand_vec_perm_shrp (struct expand_vec_perm_d *d)
11306 {
11307   unsigned i, nelt = d->nelt, shift, mask;
11308   rtx tmp, hi, lo;
11309
11310   /* ??? Don't force V2SFmode into the integer registers.  */
11311   if (d->vmode == V2SFmode)
11312     return false;
11313
11314   mask = (d->one_operand_p ? nelt - 1 : 2 * nelt - 1);
11315
11316   shift = d->perm[0];
11317   if (BYTES_BIG_ENDIAN && shift > nelt)
11318     return false;
11319
11320   for (i = 1; i < nelt; ++i)
11321     if (d->perm[i] != ((shift + i) & mask))
11322       return false;
11323
11324   if (d->testing_p)
11325     return true;
11326
11327   hi = shift < nelt ? d->op1 : d->op0;
11328   lo = shift < nelt ? d->op0 : d->op1;
11329
11330   shift %= nelt;
11331
11332   shift *= GET_MODE_UNIT_SIZE (d->vmode) * BITS_PER_UNIT;
11333
11334   /* We've eliminated the shift 0 case via expand_vec_perm_identity.  */
11335   gcc_assert (IN_RANGE (shift, 1, 63));
11336
11337   /* Recall that big-endian elements are numbered starting at the top of
11338      the register.  Ideally we'd have a shift-left-pair.  But since we
11339      don't, convert to a shift the other direction.  */
11340   if (BYTES_BIG_ENDIAN)
11341     shift = 64 - shift;
11342
11343   tmp = gen_reg_rtx (DImode);
11344   hi = gen_lowpart (DImode, hi);
11345   lo = gen_lowpart (DImode, lo);
11346   emit_insn (gen_shrp (tmp, hi, lo, GEN_INT (shift)));
11347
11348   emit_move_insn (d->target, gen_lowpart (d->vmode, tmp));
11349   return true;
11350 }
11351
11352 /* Try to instantiate D in a single instruction.  */
11353
11354 static bool
11355 expand_vec_perm_1 (struct expand_vec_perm_d *d)
11356 {
11357   unsigned i, nelt = d->nelt;
11358   unsigned char perm2[MAX_VECT_LEN];
11359
11360   /* Try single-operand selections.  */
11361   if (d->one_operand_p)
11362     {
11363       if (expand_vec_perm_identity (d))
11364         return true;
11365       if (expand_vselect (d->target, d->op0, d->perm, nelt))
11366         return true;
11367     }
11368
11369   /* Try two operand selections.  */
11370   if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt))
11371     return true;
11372
11373   /* Recognize interleave style patterns with reversed operands.  */
11374   if (!d->one_operand_p)
11375     {
11376       for (i = 0; i < nelt; ++i)
11377         {
11378           unsigned e = d->perm[i];
11379           if (e >= nelt)
11380             e -= nelt;
11381           else
11382             e += nelt;
11383           perm2[i] = e;
11384         }
11385
11386       if (expand_vselect_vconcat (d->target, d->op1, d->op0, perm2, nelt))
11387         return true;
11388     }
11389
11390   if (expand_vec_perm_shrp (d))
11391     return true;
11392
11393   /* ??? Look for deposit-like permutations where most of the result
11394      comes from one vector unchanged and the rest comes from a
11395      sequential hunk of the other vector.  */
11396
11397   return false;
11398 }
11399
11400 /* Pattern match broadcast permutations.  */
11401
11402 static bool
11403 expand_vec_perm_broadcast (struct expand_vec_perm_d *d)
11404 {
11405   unsigned i, elt, nelt = d->nelt;
11406   unsigned char perm2[2];
11407   rtx temp;
11408   bool ok;
11409
11410   if (!d->one_operand_p)
11411     return false;
11412
11413   elt = d->perm[0];
11414   for (i = 1; i < nelt; ++i)
11415     if (d->perm[i] != elt)
11416       return false;
11417
11418   switch (d->vmode)
11419     {
11420     case V2SImode:
11421     case V2SFmode:
11422       /* Implementable by interleave.  */
11423       perm2[0] = elt;
11424       perm2[1] = elt + 2;
11425       ok = expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, 2);
11426       gcc_assert (ok);
11427       break;
11428
11429     case V8QImode:
11430       /* Implementable by extract + broadcast.  */
11431       if (BYTES_BIG_ENDIAN)
11432         elt = 7 - elt;
11433       elt *= BITS_PER_UNIT;
11434       temp = gen_reg_rtx (DImode);
11435       emit_insn (gen_extzv (temp, gen_lowpart (DImode, d->op0),
11436                             GEN_INT (8), GEN_INT (elt)));
11437       emit_insn (gen_mux1_brcst_qi (d->target, gen_lowpart (QImode, temp)));
11438       break;
11439
11440     case V4HImode:
11441       /* Should have been matched directly by vec_select.  */
11442     default:
11443       gcc_unreachable ();
11444     }
11445
11446   return true;
11447 }
11448
11449 /* A subroutine of ia64_expand_vec_perm_const_1.  Try to simplify a
11450    two vector permutation into a single vector permutation by using
11451    an interleave operation to merge the vectors.  */
11452
11453 static bool
11454 expand_vec_perm_interleave_2 (struct expand_vec_perm_d *d)
11455 {
11456   struct expand_vec_perm_d dremap, dfinal;
11457   unsigned char remap[2 * MAX_VECT_LEN];
11458   unsigned contents, i, nelt, nelt2;
11459   unsigned h0, h1, h2, h3;
11460   rtx_insn *seq;
11461   bool ok;
11462
11463   if (d->one_operand_p)
11464     return false;
11465
11466   nelt = d->nelt;
11467   nelt2 = nelt / 2;
11468
11469   /* Examine from whence the elements come.  */
11470   contents = 0;
11471   for (i = 0; i < nelt; ++i)
11472     contents |= 1u << d->perm[i];
11473
11474   memset (remap, 0xff, sizeof (remap));
11475   dremap = *d;
11476
11477   h0 = (1u << nelt2) - 1;
11478   h1 = h0 << nelt2;
11479   h2 = h0 << nelt;
11480   h3 = h0 << (nelt + nelt2);
11481
11482   if ((contents & (h0 | h2)) == contents)       /* punpck even halves */
11483     {
11484       for (i = 0; i < nelt; ++i)
11485         {
11486           unsigned which = i / 2 + (i & 1 ? nelt : 0);
11487           remap[which] = i;
11488           dremap.perm[i] = which;
11489         }
11490     }
11491   else if ((contents & (h1 | h3)) == contents)  /* punpck odd halves */
11492     {
11493       for (i = 0; i < nelt; ++i)
11494         {
11495           unsigned which = i / 2 + nelt2 + (i & 1 ? nelt : 0);
11496           remap[which] = i;
11497           dremap.perm[i] = which;
11498         }
11499     }
11500   else if ((contents & 0x5555) == contents)     /* mix even elements */
11501     {
11502       for (i = 0; i < nelt; ++i)
11503         {
11504           unsigned which = (i & ~1) + (i & 1 ? nelt : 0);
11505           remap[which] = i;
11506           dremap.perm[i] = which;
11507         }
11508     }
11509   else if ((contents & 0xaaaa) == contents)     /* mix odd elements */
11510     {
11511       for (i = 0; i < nelt; ++i)
11512         {
11513           unsigned which = (i | 1) + (i & 1 ? nelt : 0);
11514           remap[which] = i;
11515           dremap.perm[i] = which;
11516         }
11517     }
11518   else if (floor_log2 (contents) - ctz_hwi (contents) < (int)nelt) /* shrp */
11519     {
11520       unsigned shift = ctz_hwi (contents);
11521       for (i = 0; i < nelt; ++i)
11522         {
11523           unsigned which = (i + shift) & (2 * nelt - 1);
11524           remap[which] = i;
11525           dremap.perm[i] = which;
11526         }
11527     }
11528   else
11529     return false;
11530
11531   /* Use the remapping array set up above to move the elements from their
11532      swizzled locations into their final destinations.  */
11533   dfinal = *d;
11534   for (i = 0; i < nelt; ++i)
11535     {
11536       unsigned e = remap[d->perm[i]];
11537       gcc_assert (e < nelt);
11538       dfinal.perm[i] = e;
11539     }
11540   dfinal.op0 = gen_reg_rtx (dfinal.vmode);
11541   dfinal.op1 = dfinal.op0;
11542   dfinal.one_operand_p = true;
11543   dremap.target = dfinal.op0;
11544
11545   /* Test if the final remap can be done with a single insn.  For V4HImode
11546      this *will* succeed.  For V8QImode or V2SImode it may not.  */
11547   start_sequence ();
11548   ok = expand_vec_perm_1 (&dfinal);
11549   seq = get_insns ();
11550   end_sequence ();
11551   if (!ok)
11552     return false;
11553   if (d->testing_p)
11554     return true;
11555
11556   ok = expand_vec_perm_1 (&dremap);
11557   gcc_assert (ok);
11558
11559   emit_insn (seq);
11560   return true;
11561 }
11562
11563 /* A subroutine of ia64_expand_vec_perm_const_1.  Emit a full V4HImode
11564    constant permutation via two mux2 and a merge.  */
11565
11566 static bool
11567 expand_vec_perm_v4hi_5 (struct expand_vec_perm_d *d)
11568 {
11569   unsigned char perm2[4];
11570   rtx rmask[4];
11571   unsigned i;
11572   rtx t0, t1, mask, x;
11573   bool ok;
11574
11575   if (d->vmode != V4HImode || d->one_operand_p)
11576     return false;
11577   if (d->testing_p)
11578     return true;
11579
11580   for (i = 0; i < 4; ++i)
11581     {
11582       perm2[i] = d->perm[i] & 3;
11583       rmask[i] = (d->perm[i] & 4 ? const0_rtx : constm1_rtx);
11584     }
11585   mask = gen_rtx_CONST_VECTOR (V4HImode, gen_rtvec_v (4, rmask));
11586   mask = force_reg (V4HImode, mask);
11587
11588   t0 = gen_reg_rtx (V4HImode);
11589   t1 = gen_reg_rtx (V4HImode);
11590
11591   ok = expand_vselect (t0, d->op0, perm2, 4);
11592   gcc_assert (ok);
11593   ok = expand_vselect (t1, d->op1, perm2, 4);
11594   gcc_assert (ok);
11595
11596   x = gen_rtx_AND (V4HImode, mask, t0);
11597   emit_insn (gen_rtx_SET (VOIDmode, t0, x));
11598
11599   x = gen_rtx_NOT (V4HImode, mask);
11600   x = gen_rtx_AND (V4HImode, x, t1);
11601   emit_insn (gen_rtx_SET (VOIDmode, t1, x));
11602
11603   x = gen_rtx_IOR (V4HImode, t0, t1);
11604   emit_insn (gen_rtx_SET (VOIDmode, d->target, x));
11605
11606   return true;
11607 }
11608
11609 /* The guts of ia64_expand_vec_perm_const, also used by the ok hook.
11610    With all of the interface bits taken care of, perform the expansion
11611    in D and return true on success.  */
11612
11613 static bool
11614 ia64_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
11615 {
11616   if (expand_vec_perm_1 (d))
11617     return true;
11618   if (expand_vec_perm_broadcast (d))
11619     return true;
11620   if (expand_vec_perm_interleave_2 (d))
11621     return true;
11622   if (expand_vec_perm_v4hi_5 (d))
11623     return true;
11624   return false;
11625 }
11626
11627 bool
11628 ia64_expand_vec_perm_const (rtx operands[4])
11629 {
11630   struct expand_vec_perm_d d;
11631   unsigned char perm[MAX_VECT_LEN];
11632   int i, nelt, which;
11633   rtx sel;
11634
11635   d.target = operands[0];
11636   d.op0 = operands[1];
11637   d.op1 = operands[2];
11638   sel = operands[3];
11639
11640   d.vmode = GET_MODE (d.target);
11641   gcc_assert (VECTOR_MODE_P (d.vmode));
11642   d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
11643   d.testing_p = false;
11644
11645   gcc_assert (GET_CODE (sel) == CONST_VECTOR);
11646   gcc_assert (XVECLEN (sel, 0) == nelt);
11647   gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
11648
11649   for (i = which = 0; i < nelt; ++i)
11650     {
11651       rtx e = XVECEXP (sel, 0, i);
11652       int ei = INTVAL (e) & (2 * nelt - 1);
11653
11654       which |= (ei < nelt ? 1 : 2);
11655       d.perm[i] = ei;
11656       perm[i] = ei;
11657     }
11658
11659   switch (which)
11660     {
11661     default:
11662       gcc_unreachable();
11663
11664     case 3:
11665       if (!rtx_equal_p (d.op0, d.op1))
11666         {
11667           d.one_operand_p = false;
11668           break;
11669         }
11670
11671       /* The elements of PERM do not suggest that only the first operand
11672          is used, but both operands are identical.  Allow easier matching
11673          of the permutation by folding the permutation into the single
11674          input vector.  */
11675       for (i = 0; i < nelt; ++i)
11676         if (d.perm[i] >= nelt)
11677           d.perm[i] -= nelt;
11678       /* FALLTHRU */
11679
11680     case 1:
11681       d.op1 = d.op0;
11682       d.one_operand_p = true;
11683       break;
11684
11685     case 2:
11686       for (i = 0; i < nelt; ++i)
11687         d.perm[i] -= nelt;
11688       d.op0 = d.op1;
11689       d.one_operand_p = true;
11690       break;
11691     }
11692
11693   if (ia64_expand_vec_perm_const_1 (&d))
11694     return true;
11695
11696   /* If the mask says both arguments are needed, but they are the same,
11697      the above tried to expand with one_operand_p true.  If that didn't
11698      work, retry with one_operand_p false, as that's what we used in _ok.  */
11699   if (which == 3 && d.one_operand_p)
11700     {
11701       memcpy (d.perm, perm, sizeof (perm));
11702       d.one_operand_p = false;
11703       return ia64_expand_vec_perm_const_1 (&d);
11704     }
11705
11706   return false;
11707 }
11708
11709 /* Implement targetm.vectorize.vec_perm_const_ok.  */
11710
11711 static bool
11712 ia64_vectorize_vec_perm_const_ok (machine_mode vmode,
11713                                   const unsigned char *sel)
11714 {
11715   struct expand_vec_perm_d d;
11716   unsigned int i, nelt, which;
11717   bool ret;
11718
11719   d.vmode = vmode;
11720   d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
11721   d.testing_p = true;
11722
11723   /* Extract the values from the vector CST into the permutation
11724      array in D.  */
11725   memcpy (d.perm, sel, nelt);
11726   for (i = which = 0; i < nelt; ++i)
11727     {
11728       unsigned char e = d.perm[i];
11729       gcc_assert (e < 2 * nelt);
11730       which |= (e < nelt ? 1 : 2);
11731     }
11732
11733   /* For all elements from second vector, fold the elements to first.  */
11734   if (which == 2)
11735     for (i = 0; i < nelt; ++i)
11736       d.perm[i] -= nelt;
11737
11738   /* Check whether the mask can be applied to the vector type.  */
11739   d.one_operand_p = (which != 3);
11740
11741   /* Otherwise we have to go through the motions and see if we can
11742      figure out how to generate the requested permutation.  */
11743   d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
11744   d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
11745   if (!d.one_operand_p)
11746     d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
11747
11748   start_sequence ();
11749   ret = ia64_expand_vec_perm_const_1 (&d);
11750   end_sequence ();
11751
11752   return ret;
11753 }
11754
11755 void
11756 ia64_expand_vec_setv2sf (rtx operands[3])
11757 {
11758   struct expand_vec_perm_d d;
11759   unsigned int which;
11760   bool ok;
11761
11762   d.target = operands[0];
11763   d.op0 = operands[0];
11764   d.op1 = gen_reg_rtx (V2SFmode);
11765   d.vmode = V2SFmode;
11766   d.nelt = 2;
11767   d.one_operand_p = false;
11768   d.testing_p = false;
11769
11770   which = INTVAL (operands[2]);
11771   gcc_assert (which <= 1);
11772   d.perm[0] = 1 - which;
11773   d.perm[1] = which + 2;
11774
11775   emit_insn (gen_fpack (d.op1, operands[1], CONST0_RTX (SFmode)));
11776
11777   ok = ia64_expand_vec_perm_const_1 (&d);
11778   gcc_assert (ok);
11779 }
11780
11781 void
11782 ia64_expand_vec_perm_even_odd (rtx target, rtx op0, rtx op1, int odd)
11783 {
11784   struct expand_vec_perm_d d;
11785   machine_mode vmode = GET_MODE (target);
11786   unsigned int i, nelt = GET_MODE_NUNITS (vmode);
11787   bool ok;
11788
11789   d.target = target;
11790   d.op0 = op0;
11791   d.op1 = op1;
11792   d.vmode = vmode;
11793   d.nelt = nelt;
11794   d.one_operand_p = false;
11795   d.testing_p = false;
11796
11797   for (i = 0; i < nelt; ++i)
11798     d.perm[i] = i * 2 + odd;
11799
11800   ok = ia64_expand_vec_perm_const_1 (&d);
11801   gcc_assert (ok);
11802 }
11803
11804 #include "gt-ia64.h"