/* Scalar Replacement of Aggregates (SRA) converts some structure
references into scalar references, exposing them to the scalar
optimizers.
- Copyright (C) 2008-2015 Free Software Foundation, Inc.
+ Copyright (C) 2008-2020 Free Software Foundation, Inc.
Contributed by Martin Jambor <mjambor@suse.cz>
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "alloc-pool.h"
#include "backend.h"
+#include "target.h"
+#include "rtl.h"
#include "tree.h"
#include "gimple.h"
-#include "rtl.h"
+#include "predict.h"
+#include "alloc-pool.h"
+#include "tree-pass.h"
#include "ssa.h"
+#include "cgraph.h"
+#include "gimple-pretty-print.h"
#include "alias.h"
#include "fold-const.h"
-#include "internal-fn.h"
#include "tree-eh.h"
#include "stor-layout.h"
#include "gimplify.h"
#include "gimplify-me.h"
#include "gimple-walk.h"
#include "tree-cfg.h"
-#include "flags.h"
-#include "insn-config.h"
-#include "expmed.h"
-#include "dojump.h"
-#include "explow.h"
-#include "calls.h"
-#include "emit-rtl.h"
-#include "varasm.h"
-#include "stmt.h"
-#include "expr.h"
#include "tree-dfa.h"
#include "tree-ssa.h"
-#include "tree-pass.h"
-#include "cgraph.h"
-#include "symbol-summary.h"
-#include "ipa-prop.h"
-#include "params.h"
-#include "target.h"
#include "dbgcnt.h"
-#include "tree-inline.h"
-#include "gimple-pretty-print.h"
-#include "ipa-inline.h"
-#include "ipa-utils.h"
#include "builtins.h"
+#include "tree-sra.h"
+
/* Enumeration of all aggregate reductions we can do. */
enum sra_mode { SRA_MODE_EARLY_IPA, /* early call regularization */
tree type;
/* The statement this access belongs to. */
- gimple stmt;
+ gimple *stmt;
/* Next group representative for this aggregate. */
struct access *next_grp;
the representative. */
struct access *group_representative;
+ /* After access tree has been constructed, this points to the parent of the
+ current access, if there is one. NULL for roots. */
+ struct access *parent;
+
/* If this access has any children (in terms of the definition above), this
points to the first one. */
struct access *first_child;
/* In intraprocedural SRA, pointer to the next sibling in the access tree as
- described above. In IPA-SRA this is a pointer to the next access
- belonging to the same group (having the same representative). */
+ described above. */
struct access *next_sibling;
/* Pointers to the first and last element in the linked list of assign
- links. */
- struct assign_link *first_link, *last_link;
+ links for propagation from LHS to RHS. */
+ struct assign_link *first_rhs_link, *last_rhs_link;
+
+ /* Pointers to the first and last element in the linked list of assign
+ links for propagation from LHS to RHS. */
+ struct assign_link *first_lhs_link, *last_lhs_link;
- /* Pointer to the next access in the work queue. */
- struct access *next_queued;
+ /* Pointer to the next access in the work queues. */
+ struct access *next_rhs_queued, *next_lhs_queued;
/* Replacement variable for this access "region." Never to be accessed
directly, always only by the means of get_access_replacement() and only
when grp_to_be_replaced flag is set. */
tree replacement_decl;
+ /* Is this access made in reverse storage order? */
+ unsigned reverse : 1;
+
/* Is this particular access write access? */
unsigned write : 1;
- /* Is this access an access to a non-addressable field? */
- unsigned non_addressable : 1;
+ /* Is this access currently in the rhs work queue? */
+ unsigned grp_rhs_queued : 1;
- /* Is this access currently in the work queue? */
- unsigned grp_queued : 1;
+ /* Is this access currently in the lhs work queue? */
+ unsigned grp_lhs_queued : 1;
/* Does this group contain a write access? This flag is propagated down the
access tree. */
is not propagated in the access tree in any direction. */
unsigned grp_scalar_write : 1;
- /* Is this access an artificial one created to scalarize some record
- entirely? */
+ /* In a root of an access tree, true means that the entire tree should be
+ totally scalarized - that all scalar leafs should be scalarized and
+ non-root grp_total_scalarization accesses should be honored. Otherwise,
+ non-root accesses with grp_total_scalarization should never get scalar
+ replacements. */
unsigned grp_total_scalarization : 1;
/* Other passes of the analysis use this bit to make function
access tree. */
unsigned grp_unscalarized_data : 1;
+ /* Set if all accesses in the group consist of the same chain of
+ COMPONENT_REFs and ARRAY_REFs. */
+ unsigned grp_same_access_path : 1;
+
/* Does this access and/or group contain a write access through a
BIT_FIELD_REF? */
unsigned grp_partial_lhs : 1;
/* Should TREE_NO_WARNING of a replacement be set? */
unsigned grp_no_warning : 1;
-
- /* Is it possible that the group refers to data which might be (directly or
- otherwise) modified? */
- unsigned grp_maybe_modified : 1;
-
- /* Set when this is a representative of a pointer to scalar (i.e. by
- reference) parameter which we consider for turning into a plain scalar
- (i.e. a by value parameter). */
- unsigned grp_scalar_ptr : 1;
-
- /* Set when we discover that this pointer is not safe to dereference in the
- caller. */
- unsigned grp_not_necessarilly_dereferenced : 1;
-
- /* Pool allocation new operator. */
- inline void *operator new (size_t)
- {
- return pool.allocate ();
- }
-
- /* Delete operator utilizing pool allocation. */
- inline void operator delete (void *ptr)
- {
- pool.remove ((access *) ptr);
- }
-
- /* Memory allocation pool. */
- static pool_allocator<access> pool;
};
typedef struct access *access_p;
/* Alloc pool for allocating access structures. */
-pool_allocator<struct access> access::pool ("SRA accesses", 16);
+static object_allocator<struct access> access_pool ("SRA accesses");
/* A structure linking lhs and rhs accesses from an aggregate assignment. They
- are used to propagate subaccesses from rhs to lhs as long as they don't
- conflict with what is already there. */
+ are used to propagate subaccesses from rhs to lhs and vice versa as long as
+ they don't conflict with what is already there. In the RHS->LHS direction,
+ we also propagate grp_write flag to lazily mark that the access contains any
+ meaningful data. */
struct assign_link
{
struct access *lacc, *racc;
- struct assign_link *next;
-
- /* Pool allocation new operator. */
- inline void *operator new (size_t)
- {
- return pool.allocate ();
- }
-
- /* Delete operator utilizing pool allocation. */
- inline void operator delete (void *ptr)
- {
- pool.remove ((assign_link *) ptr);
- }
-
- /* Memory allocation pool. */
- static pool_allocator<assign_link> pool;
+ struct assign_link *next_rhs, *next_lhs;
};
/* Alloc pool for allocating assign link structures. */
-pool_allocator<assign_link> assign_link::pool ("SRA links", 16);
+static object_allocator<assign_link> assign_link_pool ("SRA links");
/* Base (tree) -> Vector (vec<access_p> *) map. */
static hash_map<tree, auto_vec<access_p> > *base_access_vec;
+/* Hash to limit creation of artificial accesses */
+static hash_map<tree, unsigned> *propagation_budget;
+
/* Candidate hash table helpers. */
struct uid_decl_hasher : nofree_ptr_hash <tree_node>
those which cannot be (because they are and need be used as a whole). */
static bitmap should_scalarize_away_bitmap, cannot_scalarize_away_bitmap;
+/* Bitmap of candidates in the constant pool, which cannot be scalarized
+ because this would produce non-constant expressions (e.g. Ada). */
+static bitmap disqualified_constants;
+
/* Obstack for creation of fancy names. */
static struct obstack name_obstack;
/* Head of a linked list of accesses that need to have its subaccesses
propagated to their assignment counterparts. */
-static struct access *work_queue_head;
-
-/* Number of parameters of the analyzed function when doing early ipa SRA. */
-static int func_param_count;
-
-/* scan_function sets the following to true if it encounters a call to
- __builtin_apply_args. */
-static bool encountered_apply_args;
-
-/* Set by scan_function when it finds a recursive call. */
-static bool encountered_recursive_call;
-
-/* Set by scan_function when it finds a recursive call with less actual
- arguments than formal parameters.. */
-static bool encountered_unchangable_recursive_call;
-
-/* This is a table in which for each basic block and parameter there is a
- distance (offset + size) in that parameter which is dereferenced and
- accessed in that BB. */
-static HOST_WIDE_INT *bb_dereferences;
-/* Bitmap of BBs that can cause the function to "stop" progressing by
- returning, throwing externally, looping infinitely or calling a function
- which might abort etc.. */
-static bitmap final_bbs;
-
-/* Representative of no accesses at all. */
-static struct access no_accesses_representant;
-
-/* Predicate to test the special value. */
-
-static inline bool
-no_accesses_p (struct access *access)
-{
- return access == &no_accesses_representant;
-}
+static struct access *rhs_work_queue_head, *lhs_work_queue_head;
/* Dump contents of ACCESS to file F in a human friendly way. If GRP is true,
representative fields are dumped, otherwise those which only describe the
{
fprintf (f, "access { ");
fprintf (f, "base = (%d)'", DECL_UID (access->base));
- print_generic_expr (f, access->base, 0);
+ print_generic_expr (f, access->base);
fprintf (f, "', offset = " HOST_WIDE_INT_PRINT_DEC, access->offset);
fprintf (f, ", size = " HOST_WIDE_INT_PRINT_DEC, access->size);
fprintf (f, ", expr = ");
- print_generic_expr (f, access->expr, 0);
+ print_generic_expr (f, access->expr);
fprintf (f, ", type = ");
- print_generic_expr (f, access->type, 0);
+ print_generic_expr (f, access->type);
+ fprintf (f, ", reverse = %d", access->reverse);
if (grp)
fprintf (f, ", grp_read = %d, grp_write = %d, grp_assignment_read = %d, "
"grp_assignment_write = %d, grp_scalar_read = %d, "
"grp_scalar_write = %d, grp_total_scalarization = %d, "
"grp_hint = %d, grp_covered = %d, "
"grp_unscalarizable_region = %d, grp_unscalarized_data = %d, "
- "grp_partial_lhs = %d, grp_to_be_replaced = %d, "
- "grp_to_be_debug_replaced = %d, grp_maybe_modified = %d, "
- "grp_not_necessarilly_dereferenced = %d\n",
+ "grp_same_access_path = %d, grp_partial_lhs = %d, "
+ "grp_to_be_replaced = %d, grp_to_be_debug_replaced = %d}\n",
access->grp_read, access->grp_write, access->grp_assignment_read,
access->grp_assignment_write, access->grp_scalar_read,
access->grp_scalar_write, access->grp_total_scalarization,
access->grp_hint, access->grp_covered,
access->grp_unscalarizable_region, access->grp_unscalarized_data,
- access->grp_partial_lhs, access->grp_to_be_replaced,
- access->grp_to_be_debug_replaced, access->grp_maybe_modified,
- access->grp_not_necessarilly_dereferenced);
+ access->grp_same_access_path, access->grp_partial_lhs,
+ access->grp_to_be_replaced, access->grp_to_be_debug_replaced);
else
fprintf (f, ", write = %d, grp_total_scalarization = %d, "
- "grp_partial_lhs = %d\n",
+ "grp_partial_lhs = %d}\n",
access->write, access->grp_total_scalarization,
access->grp_partial_lhs);
}
int i;
for (i = 0; i < level; i++)
- fputs ("* ", dump_file);
+ fputs ("* ", f);
dump_access (f, access, true);
access = child;
}
+ /* Total scalarization does not replace single field structures with their
+ single field but rather creates an access for them underneath. Look for
+ it. */
+ if (access)
+ while (access->first_child
+ && access->first_child->offset == offset
+ && access->first_child->size == size)
+ access = access->first_child;
+
return access;
}
}
/* Add LINK to the linked list of assign links of RACC. */
+
static void
add_link_to_rhs (struct access *racc, struct assign_link *link)
{
gcc_assert (link->racc == racc);
- if (!racc->first_link)
+ if (!racc->first_rhs_link)
{
- gcc_assert (!racc->last_link);
- racc->first_link = link;
+ gcc_assert (!racc->last_rhs_link);
+ racc->first_rhs_link = link;
}
else
- racc->last_link->next = link;
+ racc->last_rhs_link->next_rhs = link;
- racc->last_link = link;
- link->next = NULL;
+ racc->last_rhs_link = link;
+ link->next_rhs = NULL;
}
-/* Move all link structures in their linked list in OLD_RACC to the linked list
- in NEW_RACC. */
+/* Add LINK to the linked list of lhs assign links of LACC. */
+
static void
-relink_to_new_repr (struct access *new_racc, struct access *old_racc)
+add_link_to_lhs (struct access *lacc, struct assign_link *link)
{
- if (!old_racc->first_link)
+ gcc_assert (link->lacc == lacc);
+
+ if (!lacc->first_lhs_link)
{
- gcc_assert (!old_racc->last_link);
- return;
+ gcc_assert (!lacc->last_lhs_link);
+ lacc->first_lhs_link = link;
}
+ else
+ lacc->last_lhs_link->next_lhs = link;
+
+ lacc->last_lhs_link = link;
+ link->next_lhs = NULL;
+}
- if (new_racc->first_link)
+/* Move all link structures in their linked list in OLD_ACC to the linked list
+ in NEW_ACC. */
+static void
+relink_to_new_repr (struct access *new_acc, struct access *old_acc)
+{
+ if (old_acc->first_rhs_link)
{
- gcc_assert (!new_racc->last_link->next);
- gcc_assert (!old_racc->last_link || !old_racc->last_link->next);
- new_racc->last_link->next = old_racc->first_link;
- new_racc->last_link = old_racc->last_link;
+ if (new_acc->first_rhs_link)
+ {
+ gcc_assert (!new_acc->last_rhs_link->next_rhs);
+ gcc_assert (!old_acc->last_rhs_link
+ || !old_acc->last_rhs_link->next_rhs);
+
+ new_acc->last_rhs_link->next_rhs = old_acc->first_rhs_link;
+ new_acc->last_rhs_link = old_acc->last_rhs_link;
+ }
+ else
+ {
+ gcc_assert (!new_acc->last_rhs_link);
+
+ new_acc->first_rhs_link = old_acc->first_rhs_link;
+ new_acc->last_rhs_link = old_acc->last_rhs_link;
+ }
+ old_acc->first_rhs_link = old_acc->last_rhs_link = NULL;
}
else
+ gcc_assert (!old_acc->last_rhs_link);
+
+ if (old_acc->first_lhs_link)
{
- gcc_assert (!new_racc->last_link);
- new_racc->first_link = old_racc->first_link;
- new_racc->last_link = old_racc->last_link;
+ if (new_acc->first_lhs_link)
+ {
+ gcc_assert (!new_acc->last_lhs_link->next_lhs);
+ gcc_assert (!old_acc->last_lhs_link
+ || !old_acc->last_lhs_link->next_lhs);
+
+ new_acc->last_lhs_link->next_lhs = old_acc->first_lhs_link;
+ new_acc->last_lhs_link = old_acc->last_lhs_link;
+ }
+ else
+ {
+ gcc_assert (!new_acc->last_lhs_link);
+
+ new_acc->first_lhs_link = old_acc->first_lhs_link;
+ new_acc->last_lhs_link = old_acc->last_lhs_link;
+ }
+ old_acc->first_lhs_link = old_acc->last_lhs_link = NULL;
+ }
+ else
+ gcc_assert (!old_acc->last_lhs_link);
+
+}
+
+/* Add ACCESS to the work to queue for propagation of subaccesses from RHS to
+ LHS (which is actually a stack). */
+
+static void
+add_access_to_rhs_work_queue (struct access *access)
+{
+ if (access->first_rhs_link && !access->grp_rhs_queued)
+ {
+ gcc_assert (!access->next_rhs_queued);
+ access->next_rhs_queued = rhs_work_queue_head;
+ access->grp_rhs_queued = 1;
+ rhs_work_queue_head = access;
}
- old_racc->first_link = old_racc->last_link = NULL;
}
-/* Add ACCESS to the work queue (which is actually a stack). */
+/* Add ACCESS to the work to queue for propagation of subaccesses from LHS to
+ RHS (which is actually a stack). */
static void
-add_access_to_work_queue (struct access *access)
+add_access_to_lhs_work_queue (struct access *access)
{
- if (!access->grp_queued)
+ if (access->first_lhs_link && !access->grp_lhs_queued)
{
- gcc_assert (!access->next_queued);
- access->next_queued = work_queue_head;
- access->grp_queued = 1;
- work_queue_head = access;
+ gcc_assert (!access->next_lhs_queued);
+ access->next_lhs_queued = lhs_work_queue_head;
+ access->grp_lhs_queued = 1;
+ lhs_work_queue_head = access;
}
}
-/* Pop an access from the work queue, and return it, assuming there is one. */
+/* Pop an access from the work queue for propagating from RHS to LHS, and
+ return it, assuming there is one. */
static struct access *
-pop_access_from_work_queue (void)
+pop_access_from_rhs_work_queue (void)
{
- struct access *access = work_queue_head;
+ struct access *access = rhs_work_queue_head;
- work_queue_head = access->next_queued;
- access->next_queued = NULL;
- access->grp_queued = 0;
+ rhs_work_queue_head = access->next_rhs_queued;
+ access->next_rhs_queued = NULL;
+ access->grp_rhs_queued = 0;
return access;
}
+/* Pop an access from the work queue for propagating from LHS to RHS, and
+ return it, assuming there is one. */
+
+static struct access *
+pop_access_from_lhs_work_queue (void)
+{
+ struct access *access = lhs_work_queue_head;
+
+ lhs_work_queue_head = access->next_lhs_queued;
+ access->next_lhs_queued = NULL;
+ access->grp_lhs_queued = 0;
+ return access;
+}
/* Allocate necessary structures. */
(vec_safe_length (cfun->local_decls) / 2);
should_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
cannot_scalarize_away_bitmap = BITMAP_ALLOC (NULL);
+ disqualified_constants = BITMAP_ALLOC (NULL);
gcc_obstack_init (&name_obstack);
base_access_vec = new hash_map<tree, auto_vec<access_p> >;
memset (&sra_stats, 0, sizeof (sra_stats));
- encountered_apply_args = false;
- encountered_recursive_call = false;
- encountered_unchangable_recursive_call = false;
}
/* Deallocate all general structures. */
candidates = NULL;
BITMAP_FREE (should_scalarize_away_bitmap);
BITMAP_FREE (cannot_scalarize_away_bitmap);
- access::pool.release ();
- assign_link::pool.release ();
+ BITMAP_FREE (disqualified_constants);
+ access_pool.release ();
+ assign_link_pool.release ();
obstack_free (&name_obstack, NULL);
delete base_access_vec;
}
+/* Return true if DECL is a VAR_DECL in the constant pool, false otherwise. */
+
+static bool constant_decl_p (tree decl)
+{
+ return VAR_P (decl) && DECL_IN_CONSTANT_POOL (decl);
+}
+
/* Remove DECL from candidates for SRA and write REASON to the dump file if
there is one. */
+
static void
disqualify_candidate (tree decl, const char *reason)
{
if (bitmap_clear_bit (candidate_bitmap, DECL_UID (decl)))
candidates->remove_elt_with_hash (decl, DECL_UID (decl));
+ if (constant_decl_p (decl))
+ bitmap_set_bit (disqualified_constants, DECL_UID (decl));
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "! Disqualifying ");
- print_generic_expr (dump_file, decl, 0);
+ print_generic_expr (dump_file, decl);
fprintf (dump_file, " - %s\n", reason);
}
}
/* Return true iff the type contains a field or an element which does not allow
- scalarization. */
+ scalarization. Use VISITED_TYPES to avoid re-checking already checked
+ (sub-)types. */
static bool
-type_internals_preclude_sra_p (tree type, const char **msg)
+type_internals_preclude_sra_p_1 (tree type, const char **msg,
+ hash_set<tree> *visited_types)
{
tree fld;
tree et;
+ if (visited_types->contains (type))
+ return false;
+ visited_types->add (type);
+
switch (TREE_CODE (type))
{
case RECORD_TYPE:
for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
if (TREE_CODE (fld) == FIELD_DECL)
{
+ if (TREE_CODE (fld) == FUNCTION_DECL)
+ continue;
tree ft = TREE_TYPE (fld);
if (TREE_THIS_VOLATILE (fld))
return true;
}
- if (AGGREGATE_TYPE_P (ft) && type_internals_preclude_sra_p (ft, msg))
+ if (AGGREGATE_TYPE_P (ft)
+ && type_internals_preclude_sra_p_1 (ft, msg, visited_types))
return true;
}
return true;
}
- if (AGGREGATE_TYPE_P (et) && type_internals_preclude_sra_p (et, msg))
+ if (AGGREGATE_TYPE_P (et)
+ && type_internals_preclude_sra_p_1 (et, msg, visited_types))
return true;
return false;
}
}
-/* If T is an SSA_NAME, return NULL if it is not a default def or return its
- base variable if it is. Return T if it is not an SSA_NAME. */
+/* Return true iff the type contains a field or an element which does not allow
+ scalarization. */
-static tree
-get_ssa_base_param (tree t)
+bool
+type_internals_preclude_sra_p (tree type, const char **msg)
{
- if (TREE_CODE (t) == SSA_NAME)
- {
- if (SSA_NAME_IS_DEFAULT_DEF (t))
- return SSA_NAME_VAR (t);
- else
- return NULL_TREE;
- }
- return t;
+ hash_set<tree> visited_types;
+ return type_internals_preclude_sra_p_1 (type, msg, &visited_types);
}
-/* Mark a dereference of BASE of distance DIST in a basic block tht STMT
- belongs to, unless the BB has already been marked as a potentially
- final. */
-
-static void
-mark_parm_dereference (tree base, HOST_WIDE_INT dist, gimple stmt)
-{
- basic_block bb = gimple_bb (stmt);
- int idx, parm_index = 0;
- tree parm;
-
- if (bitmap_bit_p (final_bbs, bb->index))
- return;
-
- for (parm = DECL_ARGUMENTS (current_function_decl);
- parm && parm != base;
- parm = DECL_CHAIN (parm))
- parm_index++;
-
- gcc_assert (parm_index < func_param_count);
-
- idx = bb->index * func_param_count + parm_index;
- if (bb_dereferences[idx] < dist)
- bb_dereferences[idx] = dist;
-}
/* Allocate an access structure for BASE, OFFSET and SIZE, clear it, fill in
the three fields. Also add it to the vector of accesses corresponding to
static struct access *
create_access_1 (tree base, HOST_WIDE_INT offset, HOST_WIDE_INT size)
{
- struct access *access = new struct access ();
+ struct access *access = access_pool.allocate ();
memset (access, 0, sizeof (struct access));
access->base = base;
return access;
}
+static bool maybe_add_sra_candidate (tree);
+
/* Create and insert access for EXPR. Return created access, or NULL if it is
- not possible. */
+ not possible. Also scan for uses of constant pool as we go along and add
+ to candidates. */
static struct access *
-create_access (tree expr, gimple stmt, bool write)
+create_access (tree expr, gimple *stmt, bool write)
{
struct access *access;
- HOST_WIDE_INT offset, size, max_size;
+ poly_int64 poffset, psize, pmax_size;
tree base = expr;
- bool ptr, unscalarizable_region = false;
+ bool reverse, unscalarizable_region = false;
- base = get_ref_base_and_extent (expr, &offset, &size, &max_size);
+ base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size,
+ &reverse);
- if (sra_mode == SRA_MODE_EARLY_IPA
- && TREE_CODE (base) == MEM_REF)
+ /* For constant-pool entries, check we can substitute the constant value. */
+ if (constant_decl_p (base))
{
- base = get_ssa_base_param (TREE_OPERAND (base, 0));
- if (!base)
- return NULL;
- ptr = true;
+ gcc_assert (!bitmap_bit_p (disqualified_constants, DECL_UID (base)));
+ if (expr != base
+ && !is_gimple_reg_type (TREE_TYPE (expr))
+ && dump_file && (dump_flags & TDF_DETAILS))
+ {
+ /* This occurs in Ada with accesses to ARRAY_RANGE_REFs,
+ and elements of multidimensional arrays (which are
+ multi-element arrays in their own right). */
+ fprintf (dump_file, "Allowing non-reg-type load of part"
+ " of constant-pool entry: ");
+ print_generic_expr (dump_file, expr);
+ }
+ maybe_add_sra_candidate (base);
}
- else
- ptr = false;
if (!DECL_P (base) || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
return NULL;
- if (sra_mode == SRA_MODE_EARLY_IPA)
+ HOST_WIDE_INT offset, size, max_size;
+ if (!poffset.is_constant (&offset)
+ || !psize.is_constant (&size)
+ || !pmax_size.is_constant (&max_size))
{
- if (size < 0 || size != max_size)
- {
- disqualify_candidate (base, "Encountered a variable sized access.");
- return NULL;
- }
- if (TREE_CODE (expr) == COMPONENT_REF
- && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
- {
- disqualify_candidate (base, "Encountered a bit-field access.");
- return NULL;
- }
- gcc_checking_assert ((offset % BITS_PER_UNIT) == 0);
+ disqualify_candidate (base, "Encountered a polynomial-sized access.");
+ return NULL;
+ }
- if (ptr)
- mark_parm_dereference (base, offset + size, stmt);
+ if (size != max_size)
+ {
+ size = max_size;
+ unscalarizable_region = true;
}
- else
+ if (size == 0)
+ return NULL;
+ if (size < 0)
{
- if (size != max_size)
- {
- size = max_size;
- unscalarizable_region = true;
- }
- if (size < 0)
- {
- disqualify_candidate (base, "Encountered an unconstrained access.");
- return NULL;
- }
+ disqualify_candidate (base, "Encountered an unconstrained access.");
+ return NULL;
}
access = create_access_1 (base, offset, size);
access->write = write;
access->grp_unscalarizable_region = unscalarizable_region;
access->stmt = stmt;
-
- if (TREE_CODE (expr) == COMPONENT_REF
- && DECL_NONADDRESSABLE_P (TREE_OPERAND (expr, 1)))
- access->non_addressable = 1;
+ access->reverse = reverse;
return access;
}
-/* Return true iff TYPE is a RECORD_TYPE with fields that are either of gimple
- register types or (recursively) records with only these two kinds of fields.
- It also returns false if any of these records contains a bit-field. */
+/* Return true iff TYPE is scalarizable - i.e. a RECORD_TYPE or fixed-length
+ ARRAY_TYPE with fields that are either of gimple register types (excluding
+ bit-fields) or (recursively) scalarizable types. CONST_DECL must be true if
+ we are considering a decl from constant pool. If it is false, char arrays
+ will be refused. */
static bool
-type_consists_of_records_p (tree type)
+scalarizable_type_p (tree type, bool const_decl)
{
- tree fld;
-
- if (TREE_CODE (type) != RECORD_TYPE)
+ if (is_gimple_reg_type (type))
+ return true;
+ if (type_contains_placeholder_p (type))
return false;
- for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
- if (TREE_CODE (fld) == FIELD_DECL)
- {
- tree ft = TREE_TYPE (fld);
-
- if (DECL_BIT_FIELD (fld))
- return false;
+ bool have_predecessor_field = false;
+ HOST_WIDE_INT prev_pos = 0;
- if (!is_gimple_reg_type (ft)
- && !type_consists_of_records_p (ft))
- return false;
- }
+ switch (TREE_CODE (type))
+ {
+ case RECORD_TYPE:
+ for (tree fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
+ if (TREE_CODE (fld) == FIELD_DECL)
+ {
+ tree ft = TREE_TYPE (fld);
- return true;
-}
+ if (zerop (DECL_SIZE (fld)))
+ continue;
-/* Create total_scalarization accesses for all scalar type fields in DECL that
- must be of a RECORD_TYPE conforming to type_consists_of_records_p. BASE
- must be the top-most VAR_DECL representing the variable, OFFSET must be the
- offset of DECL within BASE. REF must be the memory reference expression for
- the given decl. */
+ HOST_WIDE_INT pos = int_bit_position (fld);
+ if (have_predecessor_field
+ && pos <= prev_pos)
+ return false;
-static void
-completely_scalarize_record (tree base, tree decl, HOST_WIDE_INT offset,
- tree ref)
-{
- tree fld, decl_type = TREE_TYPE (decl);
+ have_predecessor_field = true;
+ prev_pos = pos;
- for (fld = TYPE_FIELDS (decl_type); fld; fld = DECL_CHAIN (fld))
- if (TREE_CODE (fld) == FIELD_DECL)
- {
- HOST_WIDE_INT pos = offset + int_bit_position (fld);
- tree ft = TREE_TYPE (fld);
- tree nref = build3 (COMPONENT_REF, TREE_TYPE (fld), ref, fld,
- NULL_TREE);
+ if (DECL_BIT_FIELD (fld))
+ return false;
- if (is_gimple_reg_type (ft))
- {
- struct access *access;
- HOST_WIDE_INT size;
-
- size = tree_to_uhwi (DECL_SIZE (fld));
- access = create_access_1 (base, pos, size);
- access->expr = nref;
- access->type = ft;
- access->grp_total_scalarization = 1;
- /* Accesses for intraprocedural SRA can have their stmt NULL. */
- }
- else
- completely_scalarize_record (base, fld, pos, nref);
- }
-}
+ if (!scalarizable_type_p (ft, const_decl))
+ return false;
+ }
-/* Create total_scalarization accesses for all scalar type fields in VAR and
- for VAR a a whole. VAR must be of a RECORD_TYPE conforming to
- type_consists_of_records_p. */
+ return true;
-static void
-completely_scalarize_var (tree var)
-{
- HOST_WIDE_INT size = tree_to_uhwi (DECL_SIZE (var));
- struct access *access;
+ case ARRAY_TYPE:
+ {
+ HOST_WIDE_INT min_elem_size;
+ if (const_decl)
+ min_elem_size = 0;
+ else
+ min_elem_size = BITS_PER_UNIT;
- access = create_access_1 (var, 0, size);
- access->expr = var;
- access->type = TREE_TYPE (var);
- access->grp_total_scalarization = 1;
+ if (TYPE_DOMAIN (type) == NULL_TREE
+ || !tree_fits_shwi_p (TYPE_SIZE (type))
+ || !tree_fits_shwi_p (TYPE_SIZE (TREE_TYPE (type)))
+ || (tree_to_shwi (TYPE_SIZE (TREE_TYPE (type))) <= min_elem_size)
+ || !tree_fits_shwi_p (TYPE_MIN_VALUE (TYPE_DOMAIN (type))))
+ return false;
+ if (tree_to_shwi (TYPE_SIZE (type)) == 0
+ && TYPE_MAX_VALUE (TYPE_DOMAIN (type)) == NULL_TREE)
+ /* Zero-element array, should not prevent scalarization. */
+ ;
+ else if ((tree_to_shwi (TYPE_SIZE (type)) <= 0)
+ || !tree_fits_shwi_p (TYPE_MAX_VALUE (TYPE_DOMAIN (type))))
+ /* Variable-length array, do not allow scalarization. */
+ return false;
- completely_scalarize_record (var, var, 0, var);
+ tree elem = TREE_TYPE (type);
+ if (!scalarizable_type_p (elem, const_decl))
+ return false;
+ return true;
+ }
+ default:
+ return false;
+ }
}
/* Return true if REF has an VIEW_CONVERT_EXPR somewhere in it. */
return false;
}
+/* Return true if REF contains a VIEW_CONVERT_EXPR or a COMPONENT_REF with a
+ bit-field field declaration. If TYPE_CHANGING_P is non-NULL, set the bool
+ it points to will be set if REF contains any of the above or a MEM_REF
+ expression that effectively performs type conversion. */
+
+static bool
+contains_vce_or_bfcref_p (const_tree ref, bool *type_changing_p = NULL)
+{
+ while (handled_component_p (ref))
+ {
+ if (TREE_CODE (ref) == VIEW_CONVERT_EXPR
+ || (TREE_CODE (ref) == COMPONENT_REF
+ && DECL_BIT_FIELD (TREE_OPERAND (ref, 1))))
+ {
+ if (type_changing_p)
+ *type_changing_p = true;
+ return true;
+ }
+ ref = TREE_OPERAND (ref, 0);
+ }
+
+ if (!type_changing_p
+ || TREE_CODE (ref) != MEM_REF
+ || TREE_CODE (TREE_OPERAND (ref, 0)) != ADDR_EXPR)
+ return false;
+
+ tree mem = TREE_OPERAND (TREE_OPERAND (ref, 0), 0);
+ if (TYPE_MAIN_VARIANT (TREE_TYPE (ref))
+ != TYPE_MAIN_VARIANT (TREE_TYPE (mem)))
+ *type_changing_p = true;
+
+ return false;
+}
+
/* Search the given tree for a declaration by skipping handled components and
exclude it from the candidates. */
disqualify_base_of_expr (tree t, const char *reason)
{
t = get_base_address (t);
- if (sra_mode == SRA_MODE_EARLY_IPA
- && TREE_CODE (t) == MEM_REF)
- t = get_ssa_base_param (TREE_OPERAND (t, 0));
-
if (t && DECL_P (t))
disqualify_candidate (t, reason);
}
created. */
static struct access *
-build_access_from_expr_1 (tree expr, gimple stmt, bool write)
+build_access_from_expr_1 (tree expr, gimple *stmt, bool write)
{
struct access *ret = NULL;
bool partial_ref;
else
partial_ref = false;
+ if (storage_order_barrier_p (expr))
+ {
+ disqualify_base_of_expr (expr, "storage order barrier.");
+ return NULL;
+ }
+
/* We need to dive through V_C_Es in order to get the size of its parameter
and not the result type. Ada produces such statements. We are also
capable of handling the topmost V_C_E but not any of those buried in other
switch (TREE_CODE (expr))
{
case MEM_REF:
- if (TREE_CODE (TREE_OPERAND (expr, 0)) != ADDR_EXPR
- && sra_mode != SRA_MODE_EARLY_IPA)
+ if (TREE_CODE (TREE_OPERAND (expr, 0)) != ADDR_EXPR)
return NULL;
/* fall through */
case VAR_DECL:
true if the expression is a store and false otherwise. */
static bool
-build_access_from_expr (tree expr, gimple stmt, bool write)
+build_access_from_expr (tree expr, gimple *stmt, bool write)
{
struct access *access;
NULL, in that case ignore it. */
static bool
-disqualify_if_bad_bb_terminating_stmt (gimple stmt, tree lhs, tree rhs)
+disqualify_if_bad_bb_terminating_stmt (gimple *stmt, tree lhs, tree rhs)
{
- if ((sra_mode == SRA_MODE_EARLY_INTRA || sra_mode == SRA_MODE_INTRA)
- && stmt_ends_bb_p (stmt))
+ if (stmt_ends_bb_p (stmt))
{
if (single_non_eh_succ (gimple_bb (stmt)))
return false;
return false;
}
+/* Return true if the nature of BASE is such that it contains data even if
+ there is no write to it in the function. */
+
+static bool
+comes_initialized_p (tree base)
+{
+ return TREE_CODE (base) == PARM_DECL || constant_decl_p (base);
+}
+
/* Scan expressions occurring in STMT, create access structures for all accesses
to candidates for scalarization and remove those candidates which occur in
statements or expressions that prevent them from being split apart. Return
true if any access has been inserted. */
static bool
-build_accesses_from_assign (gimple stmt)
+build_accesses_from_assign (gimple *stmt)
{
tree lhs, rhs;
struct access *lacc, *racc;
lacc = build_access_from_expr_1 (lhs, stmt, true);
if (lacc)
- lacc->grp_assignment_write = 1;
+ {
+ lacc->grp_assignment_write = 1;
+ if (storage_order_barrier_p (rhs))
+ lacc->grp_unscalarizable_region = 1;
+
+ if (should_scalarize_away_bitmap && !is_gimple_reg_type (lacc->type))
+ {
+ bool type_changing_p = false;
+ contains_vce_or_bfcref_p (lhs, &type_changing_p);
+ if (type_changing_p)
+ bitmap_set_bit (cannot_scalarize_away_bitmap,
+ DECL_UID (lacc->base));
+ }
+ }
if (racc)
{
racc->grp_assignment_read = 1;
- if (should_scalarize_away_bitmap && !gimple_has_volatile_ops (stmt)
- && !is_gimple_reg_type (racc->type))
- bitmap_set_bit (should_scalarize_away_bitmap, DECL_UID (racc->base));
+ if (should_scalarize_away_bitmap && !is_gimple_reg_type (racc->type))
+ {
+ bool type_changing_p = false;
+ contains_vce_or_bfcref_p (rhs, &type_changing_p);
+
+ if (type_changing_p || gimple_has_volatile_ops (stmt))
+ bitmap_set_bit (cannot_scalarize_away_bitmap,
+ DECL_UID (racc->base));
+ else
+ bitmap_set_bit (should_scalarize_away_bitmap,
+ DECL_UID (racc->base));
+ }
+ if (storage_order_barrier_p (lhs))
+ racc->grp_unscalarizable_region = 1;
}
if (lacc && racc
{
struct assign_link *link;
- link = new assign_link;
+ link = assign_link_pool.allocate ();
memset (link, 0, sizeof (struct assign_link));
link->lacc = lacc;
link->racc = racc;
-
add_link_to_rhs (racc, link);
+ add_link_to_lhs (lacc, link);
+ add_access_to_rhs_work_queue (racc);
+ add_access_to_lhs_work_queue (lacc);
+
+ /* Let's delay marking the areas as written until propagation of accesses
+ across link, unless the nature of rhs tells us that its data comes
+ from elsewhere. */
+ if (!comes_initialized_p (racc->base))
+ lacc->write = false;
}
return lacc || racc;
GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
static bool
-asm_visit_addr (gimple, tree op, tree, void *)
+asm_visit_addr (gimple *, tree op, tree, void *)
{
op = get_base_address (op);
if (op
return false;
}
-/* Return true iff callsite CALL has at least as many actual arguments as there
- are formal parameters of the function currently processed by IPA-SRA and
- that their types match. */
-
-static inline bool
-callsite_arguments_match_p (gimple call)
-{
- if (gimple_call_num_args (call) < (unsigned) func_param_count)
- return false;
-
- tree parm;
- int i;
- for (parm = DECL_ARGUMENTS (current_function_decl), i = 0;
- parm;
- parm = DECL_CHAIN (parm), i++)
- {
- tree arg = gimple_call_arg (call, i);
- if (!useless_type_conversion_p (TREE_TYPE (parm), TREE_TYPE (arg)))
- return false;
- }
- return true;
-}
-
/* Scan function and look for interesting expressions and create access
structures for them. Return true iff any access is created. */
gimple_stmt_iterator gsi;
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
tree t;
unsigned i;
- if (final_bbs && stmt_can_throw_external (stmt))
- bitmap_set_bit (final_bbs, bb->index);
switch (gimple_code (stmt))
{
case GIMPLE_RETURN:
t = gimple_return_retval (as_a <greturn *> (stmt));
if (t != NULL_TREE)
ret |= build_access_from_expr (t, stmt, false);
- if (final_bbs)
- bitmap_set_bit (final_bbs, bb->index);
break;
case GIMPLE_ASSIGN:
ret |= build_access_from_expr (gimple_call_arg (stmt, i),
stmt, false);
- if (sra_mode == SRA_MODE_EARLY_IPA)
- {
- tree dest = gimple_call_fndecl (stmt);
- int flags = gimple_call_flags (stmt);
-
- if (dest)
- {
- if (DECL_BUILT_IN_CLASS (dest) == BUILT_IN_NORMAL
- && DECL_FUNCTION_CODE (dest) == BUILT_IN_APPLY_ARGS)
- encountered_apply_args = true;
- if (recursive_call_p (current_function_decl, dest))
- {
- encountered_recursive_call = true;
- if (!callsite_arguments_match_p (stmt))
- encountered_unchangable_recursive_call = true;
- }
- }
-
- if (final_bbs
- && (flags & (ECF_CONST | ECF_PURE)) == 0)
- bitmap_set_bit (final_bbs, bb->index);
- }
-
t = gimple_call_lhs (stmt);
if (t && !disqualify_if_bad_bb_terminating_stmt (stmt, t, NULL))
ret |= build_access_from_expr (t, stmt, true);
gasm *asm_stmt = as_a <gasm *> (stmt);
walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
asm_visit_addr);
- if (final_bbs)
- bitmap_set_bit (final_bbs, bb->index);
-
for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
{
t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
&& TREE_CODE (f2->type) != COMPLEX_TYPE
&& TREE_CODE (f2->type) != VECTOR_TYPE)
return -1;
- /* Put the integral type with the bigger precision first. */
+ /* Put any integral type before any non-integral type. When splicing, we
+ make sure that those with insufficient precision and occupying the
+ same space are not scalarized. */
else if (INTEGRAL_TYPE_P (f1->type)
+ && !INTEGRAL_TYPE_P (f2->type))
+ return -1;
+ else if (!INTEGRAL_TYPE_P (f1->type)
&& INTEGRAL_TYPE_P (f2->type))
- return TYPE_PRECISION (f2->type) - TYPE_PRECISION (f1->type);
- /* Put any integral type with non-full precision last. */
- else if (INTEGRAL_TYPE_P (f1->type)
- && (TREE_INT_CST_LOW (TYPE_SIZE (f1->type))
- != TYPE_PRECISION (f1->type)))
return 1;
- else if (INTEGRAL_TYPE_P (f2->type)
- && (TREE_INT_CST_LOW (TYPE_SIZE (f2->type))
- != TYPE_PRECISION (f2->type)))
- return -1;
+ /* Put the integral type with the bigger precision first. */
+ else if (INTEGRAL_TYPE_P (f1->type)
+ && INTEGRAL_TYPE_P (f2->type)
+ && (TYPE_PRECISION (f2->type) != TYPE_PRECISION (f1->type)))
+ return TYPE_PRECISION (f2->type) - TYPE_PRECISION (f1->type);
/* Stabilize the sort. */
return TYPE_UID (f1->type) - TYPE_UID (f2->type);
}
}
/* Construct a MEM_REF that would reference a part of aggregate BASE of type
- EXP_TYPE at the given OFFSET. If BASE is something for which
- get_addr_base_and_unit_offset returns NULL, gsi must be non-NULL and is used
- to insert new statements either before or below the current one as specified
- by INSERT_AFTER. This function is not capable of handling bitfields.
-
- BASE must be either a declaration or a memory reference that has correct
- alignment ifformation embeded in it (e.g. a pre-existing one in SRA). */
+ EXP_TYPE at the given OFFSET and with storage order REVERSE. If BASE is
+ something for which get_addr_base_and_unit_offset returns NULL, gsi must
+ be non-NULL and is used to insert new statements either before or below
+ the current one as specified by INSERT_AFTER. This function is not capable
+ of handling bitfields. */
tree
-build_ref_for_offset (location_t loc, tree base, HOST_WIDE_INT offset,
- tree exp_type, gimple_stmt_iterator *gsi,
+build_ref_for_offset (location_t loc, tree base, poly_int64 offset,
+ bool reverse, tree exp_type, gimple_stmt_iterator *gsi,
bool insert_after)
{
tree prev_base = base;
tree off;
tree mem_ref;
- HOST_WIDE_INT base_offset;
+ poly_int64 base_offset;
unsigned HOST_WIDE_INT misalign;
unsigned int align;
- gcc_checking_assert (offset % BITS_PER_UNIT == 0);
+ /* Preserve address-space information. */
+ addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (base));
+ if (as != TYPE_ADDR_SPACE (exp_type))
+ exp_type = build_qualified_type (exp_type,
+ TYPE_QUALS (exp_type)
+ | ENCODE_QUAL_ADDR_SPACE (as));
+
+ poly_int64 byte_offset = exact_div (offset, BITS_PER_UNIT);
get_object_alignment_1 (base, &align, &misalign);
base = get_addr_base_and_unit_offset (base, &base_offset);
else
gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
- off = build_int_cst (reference_alias_ptr_type (prev_base),
- offset / BITS_PER_UNIT);
+ off = build_int_cst (reference_alias_ptr_type (prev_base), byte_offset);
base = tmp;
}
else if (TREE_CODE (base) == MEM_REF)
{
off = build_int_cst (TREE_TYPE (TREE_OPERAND (base, 1)),
- base_offset + offset / BITS_PER_UNIT);
+ base_offset + byte_offset);
off = int_const_binop (PLUS_EXPR, TREE_OPERAND (base, 1), off);
base = unshare_expr (TREE_OPERAND (base, 0));
}
else
{
- off = build_int_cst (reference_alias_ptr_type (base),
- base_offset + offset / BITS_PER_UNIT);
+ off = build_int_cst (reference_alias_ptr_type (prev_base),
+ base_offset + byte_offset);
base = build_fold_addr_expr (unshare_expr (base));
}
- misalign = (misalign + offset) & (align - 1);
- if (misalign != 0)
- align = (misalign & -misalign);
+ unsigned int align_bound = known_alignment (misalign + offset);
+ if (align_bound != 0)
+ align = MIN (align, align_bound);
if (align != TYPE_ALIGN (exp_type))
exp_type = build_aligned_type (exp_type, align);
mem_ref = fold_build2_loc (loc, MEM_REF, exp_type, base, off);
+ REF_REVERSE_STORAGE_ORDER (mem_ref) = reverse;
if (TREE_THIS_VOLATILE (prev_base))
TREE_THIS_VOLATILE (mem_ref) = 1;
if (TREE_SIDE_EFFECTS (prev_base))
return mem_ref;
}
+/* Construct and return a memory reference that is equal to a portion of
+ MODEL->expr but is based on BASE. If this cannot be done, return NULL. */
+
+static tree
+build_reconstructed_reference (location_t, tree base, struct access *model)
+{
+ tree expr = model->expr, prev_expr = NULL;
+ while (!types_compatible_p (TREE_TYPE (expr), TREE_TYPE (base)))
+ {
+ if (!handled_component_p (expr))
+ return NULL_TREE;
+ prev_expr = expr;
+ expr = TREE_OPERAND (expr, 0);
+ }
+
+ /* Guard against broken VIEW_CONVERT_EXPRs... */
+ if (!prev_expr)
+ return NULL_TREE;
+
+ TREE_OPERAND (prev_expr, 0) = base;
+ tree ref = unshare_expr (model->expr);
+ TREE_OPERAND (prev_expr, 0) = expr;
+ return ref;
+}
+
/* Construct a memory reference to a part of an aggregate BASE at the given
OFFSET and of the same type as MODEL. In case this is a reference to a
bit-field, the function will replicate the last component_ref of model's
offset -= int_bit_position (fld);
exp_type = TREE_TYPE (TREE_OPERAND (model->expr, 0));
- t = build_ref_for_offset (loc, base, offset, exp_type, gsi, insert_after);
+ t = build_ref_for_offset (loc, base, offset, model->reverse, exp_type,
+ gsi, insert_after);
+ /* The flag will be set on the record type. */
+ REF_REVERSE_STORAGE_ORDER (t) = 0;
return fold_build3_loc (loc, COMPONENT_REF, TREE_TYPE (fld), t, fld,
NULL_TREE);
}
else
- return build_ref_for_offset (loc, base, offset, model->type,
- gsi, insert_after);
+ {
+ tree res;
+ if (model->grp_same_access_path
+ && !TREE_THIS_VOLATILE (base)
+ && (TYPE_ADDR_SPACE (TREE_TYPE (base))
+ == TYPE_ADDR_SPACE (TREE_TYPE (model->expr)))
+ && offset <= model->offset
+ /* build_reconstructed_reference can still fail if we have already
+ massaged BASE because of another type incompatibility. */
+ && (res = build_reconstructed_reference (loc, base, model)))
+ return res;
+ else
+ return build_ref_for_offset (loc, base, offset, model->reverse,
+ model->type, gsi, insert_after);
+ }
}
/* Attempt to build a memory reference that we could but into a gimple
build_debug_ref_for_model (location_t loc, tree base, HOST_WIDE_INT offset,
struct access *model)
{
- HOST_WIDE_INT base_offset;
+ poly_int64 base_offset;
tree off;
if (TREE_CODE (model->expr) == COMPONENT_REF
}
}
-/* Return true iff TYPE is stdarg va_list type. */
-
-static inline bool
-is_va_list_type (tree type)
-{
- return TYPE_MAIN_VARIANT (type) == TYPE_MAIN_VARIANT (va_list_type_node);
-}
-
/* Print message to dump file why a variable was rejected. */
static void
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "Rejected (%d): %s: ", DECL_UID (var), msg);
- print_generic_expr (dump_file, var, 0);
+ print_generic_expr (dump_file, var);
fprintf (dump_file, "\n");
}
}
reject (var, "not aggregate");
return false;
}
- if (needs_to_live_in_memory (var))
+ /* Allow constant-pool entries that "need to live in memory". */
+ if (needs_to_live_in_memory (var) && !constant_decl_p (var))
{
reject (var, "needs to live in memory");
return false;
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "Candidate (%d): ", DECL_UID (var));
- print_generic_expr (dump_file, var, 0);
+ print_generic_expr (dump_file, var);
fprintf (dump_file, "\n");
}
FOR_EACH_LOCAL_DECL (cfun, i, var)
{
- if (TREE_CODE (var) != VAR_DECL)
+ if (!VAR_P (var))
continue;
ret |= maybe_add_sra_candidate (var);
return ret;
}
+/* Return true if EXP is a reference chain of COMPONENT_REFs and AREAY_REFs
+ ending either with a DECL or a MEM_REF with zero offset. */
+
+static bool
+path_comparable_for_same_access (tree expr)
+{
+ while (handled_component_p (expr))
+ {
+ if (TREE_CODE (expr) == ARRAY_REF)
+ {
+ /* SSA name indices can occur here too when the array is of sie one.
+ But we cannot just re-use array_refs with SSA names elsewhere in
+ the function, so disallow non-constant indices. TODO: Remove this
+ limitation after teaching build_reconstructed_reference to replace
+ the index with the index type lower bound. */
+ if (TREE_CODE (TREE_OPERAND (expr, 1)) != INTEGER_CST)
+ return false;
+ }
+ expr = TREE_OPERAND (expr, 0);
+ }
+
+ if (TREE_CODE (expr) == MEM_REF)
+ {
+ if (!zerop (TREE_OPERAND (expr, 1)))
+ return false;
+ }
+ else
+ gcc_assert (DECL_P (expr));
+
+ return true;
+}
+
+/* Assuming that EXP1 consists of only COMPONENT_REFs and ARRAY_REFs, return
+ true if the chain of these handled components are exactly the same as EXP2
+ and the expression under them is the same DECL or an equivalent MEM_REF.
+ The reference picked by compare_access_positions must go to EXP1. */
+
+static bool
+same_access_path_p (tree exp1, tree exp2)
+{
+ if (TREE_CODE (exp1) != TREE_CODE (exp2))
+ {
+ /* Special case single-field structures loaded sometimes as the field
+ and sometimes as the structure. If the field is of a scalar type,
+ compare_access_positions will put it into exp1.
+
+ TODO: The gimple register type condition can be removed if teach
+ compare_access_positions to put inner types first. */
+ if (is_gimple_reg_type (TREE_TYPE (exp1))
+ && TREE_CODE (exp1) == COMPONENT_REF
+ && (TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (exp1, 0)))
+ == TYPE_MAIN_VARIANT (TREE_TYPE (exp2))))
+ exp1 = TREE_OPERAND (exp1, 0);
+ else
+ return false;
+ }
+
+ if (!operand_equal_p (exp1, exp2, OEP_ADDRESS_OF))
+ return false;
+
+ return true;
+}
+
/* Sort all accesses for the given variable, check for partial overlaps and
return NULL if there are any. If there are none, pick a representative for
each combination of offset and size and create a linked list out of them.
bool grp_assignment_read = access->grp_assignment_read;
bool grp_assignment_write = access->grp_assignment_write;
bool multiple_scalar_reads = false;
- bool total_scalarization = access->grp_total_scalarization;
bool grp_partial_lhs = access->grp_partial_lhs;
bool first_scalar = is_gimple_reg_type (access->type);
bool unscalarizable_region = access->grp_unscalarizable_region;
+ bool grp_same_access_path = true;
+ bool bf_non_full_precision
+ = (INTEGRAL_TYPE_P (access->type)
+ && TYPE_PRECISION (access->type) != access->size
+ && TREE_CODE (access->expr) == COMPONENT_REF
+ && DECL_BIT_FIELD (TREE_OPERAND (access->expr, 1)));
if (first || access->offset >= high)
{
gcc_assert (access->offset >= low
&& access->offset + access->size <= high);
+ grp_same_access_path = path_comparable_for_same_access (access->expr);
+
j = i + 1;
while (j < access_count)
{
grp_assignment_write |= ac2->grp_assignment_write;
grp_partial_lhs |= ac2->grp_partial_lhs;
unscalarizable_region |= ac2->grp_unscalarizable_region;
- total_scalarization |= ac2->grp_total_scalarization;
relink_to_new_repr (access, ac2);
/* If there are both aggregate-type and scalar-type accesses with
this combination of size and offset, the comparison function
should have put the scalars first. */
gcc_assert (first_scalar || !is_gimple_reg_type (ac2->type));
+ /* It also prefers integral types to non-integral. However, when the
+ precision of the selected type does not span the entire area and
+ should also be used for a non-integer (i.e. float), we must not
+ let that happen. Normally analyze_access_subtree expands the type
+ to cover the entire area but for bit-fields it doesn't. */
+ if (bf_non_full_precision && !INTEGRAL_TYPE_P (ac2->type))
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Cannot scalarize the following access "
+ "because insufficient precision integer type was "
+ "selected.\n ");
+ dump_access (dump_file, access, false);
+ }
+ unscalarizable_region = true;
+ }
+
+ if (grp_same_access_path
+ && !same_access_path_p (access->expr, ac2->expr))
+ grp_same_access_path = false;
+
ac2->group_representative = access;
j++;
}
access->grp_scalar_write = grp_scalar_write;
access->grp_assignment_read = grp_assignment_read;
access->grp_assignment_write = grp_assignment_write;
- access->grp_hint = multiple_scalar_reads || total_scalarization;
- access->grp_total_scalarization = total_scalarization;
+ access->grp_hint = multiple_scalar_reads && !constant_decl_p (var);
access->grp_partial_lhs = grp_partial_lhs;
access->grp_unscalarizable_region = unscalarizable_region;
- if (access->first_link)
- add_access_to_work_queue (access);
+ access->grp_same_access_path = grp_same_access_path;
*prev_acc_ptr = access;
prev_acc_ptr = &access->next_grp;
/* Create a variable for the given ACCESS which determines the type, name and a
few other properties. Return the variable declaration and store it also to
- ACCESS->replacement. */
+ ACCESS->replacement. REG_TREE is used when creating a declaration to base a
+ default-definition SSA name on in order to facilitate an uninitialized
+ warning. It is used instead of the actual ACCESS type if that is not of a
+ gimple register type. */
static tree
-create_access_replacement (struct access *access)
+create_access_replacement (struct access *access, tree reg_type = NULL_TREE)
{
tree repl;
+ tree type = access->type;
+ if (reg_type && !is_gimple_reg_type (type))
+ type = reg_type;
+
if (access->grp_to_be_debug_replaced)
{
repl = create_tmp_var_raw (access->type);
else
/* Drop any special alignment on the type if it's not on the main
variant. This avoids issues with weirdo ABIs like AAPCS. */
- repl = create_tmp_var (build_qualified_type
- (TYPE_MAIN_VARIANT (access->type),
- TYPE_QUALS (access->type)), "SR");
- if (TREE_CODE (access->type) == COMPLEX_TYPE
- || TREE_CODE (access->type) == VECTOR_TYPE)
- {
- if (!access->grp_partial_lhs)
- DECL_GIMPLE_REG_P (repl) = 1;
- }
- else if (access->grp_partial_lhs
- && is_gimple_reg_type (access->type))
- TREE_ADDRESSABLE (repl) = 1;
+ repl = create_tmp_var (build_qualified_type (TYPE_MAIN_VARIANT (type),
+ TYPE_QUALS (type)), "SR");
+ if (access->grp_partial_lhs
+ && is_gimple_reg_type (type))
+ DECL_NOT_GIMPLE_REG_P (repl) = 1;
DECL_SOURCE_LOCATION (repl) = DECL_SOURCE_LOCATION (access->base);
DECL_ARTIFICIAL (repl) = 1;
bool fail = false;
DECL_NAME (repl) = get_identifier (pretty_name);
+ DECL_NAMELESS (repl) = 1;
obstack_free (&name_obstack, pretty_name);
/* Get rid of any SSA_NAMEs embedded in debug_expr,
if (access->grp_to_be_debug_replaced)
{
fprintf (dump_file, "Created a debug-only replacement for ");
- print_generic_expr (dump_file, access->base, 0);
+ print_generic_expr (dump_file, access->base);
fprintf (dump_file, " offset: %u, size: %u\n",
(unsigned) access->offset, (unsigned) access->size);
}
else
{
fprintf (dump_file, "Created a replacement for ");
- print_generic_expr (dump_file, access->base, 0);
+ print_generic_expr (dump_file, access->base);
fprintf (dump_file, " offset: %u, size: %u: ",
(unsigned) access->offset, (unsigned) access->size);
- print_generic_expr (dump_file, repl, 0);
+ print_generic_expr (dump_file, repl, TDF_UID);
fprintf (dump_file, "\n");
}
}
return repl;
}
-/* Return ACCESS scalar replacement, create it if it does not exist yet. */
+/* Return ACCESS scalar replacement, which must exist. */
static inline tree
get_access_replacement (struct access *access)
else
last_child->next_sibling = *access;
last_child = *access;
+ (*access)->parent = root;
+ (*access)->grp_write |= root->grp_write;
if (!build_access_subtree (access))
return false;
return true;
}
+/* Traverse the access forest where ROOT is the first root and verify that
+ various important invariants hold true. */
+
+DEBUG_FUNCTION void
+verify_sra_access_forest (struct access *root)
+{
+ struct access *access = root;
+ tree first_base = root->base;
+ gcc_assert (DECL_P (first_base));
+ do
+ {
+ gcc_assert (access->base == first_base);
+ if (access->parent)
+ gcc_assert (access->offset >= access->parent->offset
+ && access->size <= access->parent->size);
+ if (access->next_sibling)
+ gcc_assert (access->next_sibling->offset
+ >= access->offset + access->size);
+
+ poly_int64 poffset, psize, pmax_size;
+ bool reverse;
+ tree base = get_ref_base_and_extent (access->expr, &poffset, &psize,
+ &pmax_size, &reverse);
+ HOST_WIDE_INT offset, size, max_size;
+ if (!poffset.is_constant (&offset)
+ || !psize.is_constant (&size)
+ || !pmax_size.is_constant (&max_size))
+ gcc_unreachable ();
+ gcc_assert (base == first_base);
+ gcc_assert (offset == access->offset);
+ gcc_assert (access->grp_unscalarizable_region
+ || access->grp_total_scalarization
+ || size == max_size);
+ gcc_assert (access->grp_unscalarizable_region
+ || !is_gimple_reg_type (access->type)
+ || size == access->size);
+ gcc_assert (reverse == access->reverse);
+
+ if (access->first_child)
+ {
+ gcc_assert (access->first_child->parent == access);
+ access = access->first_child;
+ }
+ else if (access->next_sibling)
+ {
+ gcc_assert (access->next_sibling->parent == access->parent);
+ access = access->next_sibling;
+ }
+ else
+ {
+ while (access->parent && !access->next_sibling)
+ access = access->parent;
+ if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ gcc_assert (access == root);
+ root = root->next_grp;
+ access = root;
+ }
+ }
+ }
+ while (access);
+}
+
+/* Verify access forests of all candidates with accesses by calling
+ verify_access_forest on each on them. */
+
+DEBUG_FUNCTION void
+verify_all_sra_access_forests (void)
+{
+ bitmap_iterator bi;
+ unsigned i;
+ EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
+ {
+ tree var = candidate (i);
+ struct access *access = get_first_repr_for_decl (var);
+ if (access)
+ {
+ gcc_assert (access->base == var);
+ verify_sra_access_forest (access);
+ }
+ }
+}
+
/* Return true if expr contains some ARRAY_REFs into a variable bounded
array. */
}
/* Analyze the subtree of accesses rooted in ROOT, scheduling replacements when
- both seeming beneficial and when ALLOW_REPLACEMENTS allows it. Also set all
- sorts of access flags appropriately along the way, notably always set
- grp_read and grp_assign_read according to MARK_READ and grp_write when
- MARK_WRITE is true.
+ both seeming beneficial and when ALLOW_REPLACEMENTS allows it. If TOTALLY
+ is set, we are totally scalarizing the aggregate. Also set all sorts of
+ access flags appropriately along the way, notably always set grp_read and
+ grp_assign_read according to MARK_READ and grp_write when MARK_WRITE is
+ true.
Creating a replacement for a scalar access is considered beneficial if its
- grp_hint is set (this means we are either attempting total scalarization or
- there is more than one direct read access) or according to the following
- table:
+ grp_hint ot TOTALLY is set (this means either that there is more than one
+ direct read access or that we are attempting total scalarization) or
+ according to the following table:
Access written to through a scalar type (once or more times)
|
static bool
analyze_access_subtree (struct access *root, struct access *parent,
- bool allow_replacements)
+ bool allow_replacements, bool totally)
{
struct access *child;
HOST_WIDE_INT limit = root->offset + root->size;
root->grp_write = 1;
if (parent->grp_assignment_write)
root->grp_assignment_write = 1;
- if (parent->grp_total_scalarization)
- root->grp_total_scalarization = 1;
+ if (!parent->grp_same_access_path)
+ root->grp_same_access_path = 0;
}
if (root->grp_unscalarizable_region)
{
hole |= covered_to < child->offset;
sth_created |= analyze_access_subtree (child, root,
- allow_replacements && !scalar);
+ allow_replacements && !scalar,
+ totally);
root->grp_unscalarized_data |= child->grp_unscalarized_data;
- root->grp_total_scalarization &= child->grp_total_scalarization;
if (child->grp_covered)
covered_to += child->size;
else
}
if (allow_replacements && scalar && !root->first_child
- && (root->grp_hint
+ && (totally || !root->grp_total_scalarization)
+ && (totally
+ || root->grp_hint
|| ((root->grp_scalar_read || root->grp_assignment_read)
&& (root->grp_scalar_write || root->grp_assignment_write))))
{
&& (root->size % BITS_PER_UNIT) == 0);
root->type = build_nonstandard_integer_type (root->size,
TYPE_UNSIGNED (rt));
- root->expr = build_ref_for_offset (UNKNOWN_LOCATION,
- root->base, root->offset,
+ root->expr = build_ref_for_offset (UNKNOWN_LOCATION, root->base,
+ root->offset, root->reverse,
root->type, NULL, false);
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "Changing the type of a replacement for ");
- print_generic_expr (dump_file, root->base, 0);
+ print_generic_expr (dump_file, root->base);
fprintf (dump_file, " offset: %u, size: %u ",
(unsigned) root->offset, (unsigned) root->size);
fprintf (dump_file, " to an integer.\n");
{
if (allow_replacements
&& scalar && !root->first_child
+ && !root->grp_total_scalarization
&& (root->grp_scalar_write || root->grp_assignment_write)
&& !bitmap_bit_p (cannot_scalarize_away_bitmap,
DECL_UID (root->base)))
gcc_checking_assert (!root->grp_scalar_read
&& !root->grp_assignment_read);
sth_created = true;
- if (MAY_HAVE_DEBUG_STMTS)
+ if (MAY_HAVE_DEBUG_BIND_STMTS)
{
root->grp_to_be_debug_replaced = 1;
root->replacement_decl = create_access_replacement (root);
if (covered_to < limit)
hole = true;
- if (scalar)
+ if (scalar || !allow_replacements)
root->grp_total_scalarization = 0;
}
- if (!hole || root->grp_total_scalarization)
+ if (!hole || totally)
root->grp_covered = 1;
- else if (root->grp_write || TREE_CODE (root->base) == PARM_DECL)
+ else if (root->grp_write || comes_initialized_p (root->base))
root->grp_unscalarized_data = 1; /* not covered and written to */
return sth_created;
}
while (access)
{
- if (analyze_access_subtree (access, NULL, true))
+ if (analyze_access_subtree (access, NULL, true,
+ access->grp_total_scalarization))
ret = true;
access = access->next_grp;
}
return ret;
}
-/* Return true iff a potential new child of LACC at offset OFFSET and with size
+/* Return true iff a potential new child of ACC at offset OFFSET and with size
SIZE would conflict with an already existing one. If exactly such a child
- already exists in LACC, store a pointer to it in EXACT_MATCH. */
+ already exists in ACC, store a pointer to it in EXACT_MATCH. */
static bool
-child_would_conflict_in_lacc (struct access *lacc, HOST_WIDE_INT norm_offset,
+child_would_conflict_in_acc (struct access *acc, HOST_WIDE_INT norm_offset,
HOST_WIDE_INT size, struct access **exact_match)
{
struct access *child;
- for (child = lacc->first_child; child; child = child->next_sibling)
+ for (child = acc->first_child; child; child = child->next_sibling)
{
if (child->offset == norm_offset && child->size == size)
{
/* Create a new child access of PARENT, with all properties just like MODEL
except for its offset and with its grp_write false and grp_read true.
- Return the new access or NULL if it cannot be created. Note that this access
- is created long after all splicing and sorting, it's not located in any
- access vector and is automatically a representative of its group. */
+ Return the new access or NULL if it cannot be created. Note that this
+ access is created long after all splicing and sorting, it's not located in
+ any access vector and is automatically a representative of its group. Set
+ the gpr_write flag of the new accesss if SET_GRP_WRITE is true. */
static struct access *
create_artificial_child_access (struct access *parent, struct access *model,
- HOST_WIDE_INT new_offset)
+ HOST_WIDE_INT new_offset,
+ bool set_grp_read, bool set_grp_write)
{
struct access **child;
tree expr = parent->base;
gcc_assert (!model->grp_unscalarizable_region);
- struct access *access = new struct access ();
+ struct access *access = access_pool.allocate ();
memset (access, 0, sizeof (struct access));
if (!build_user_friendly_ref_for_offset (&expr, TREE_TYPE (expr), new_offset,
model->type))
access->offset = new_offset;
access->size = model->size;
access->type = model->type;
- access->grp_write = true;
- access->grp_read = false;
+ access->parent = parent;
+ access->grp_read = set_grp_read;
+ access->grp_write = set_grp_write;
+ access->reverse = model->reverse;
child = &parent->first_child;
while (*child && (*child)->offset < new_offset)
}
-/* Propagate all subaccesses of RACC across an assignment link to LACC. Return
- true if any new subaccess was created. Additionally, if RACC is a scalar
- access but LACC is not, change the type of the latter, if possible. */
+/* Beginning with ACCESS, traverse its whole access subtree and mark all
+ sub-trees as written to. If any of them has not been marked so previously
+ and has assignment links leading from it, re-enqueue it. */
+
+static void
+subtree_mark_written_and_rhs_enqueue (struct access *access)
+{
+ if (access->grp_write)
+ return;
+ access->grp_write = true;
+ add_access_to_rhs_work_queue (access);
+
+ struct access *child;
+ for (child = access->first_child; child; child = child->next_sibling)
+ subtree_mark_written_and_rhs_enqueue (child);
+}
+
+/* If there is still budget to create a propagation access for DECL, return
+ true and decrement the budget. Otherwise return false. */
+
+static bool
+budget_for_propagation_access (tree decl)
+{
+ unsigned b, *p = propagation_budget->get (decl);
+ if (p)
+ b = *p;
+ else
+ b = param_sra_max_propagations;
+
+ if (b == 0)
+ return false;
+ b--;
+
+ if (b == 0 && dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "The propagation budget of ");
+ print_generic_expr (dump_file, decl);
+ fprintf (dump_file, " (UID: %u) has been exhausted.\n", DECL_UID (decl));
+ }
+ propagation_budget->put (decl, b);
+ return true;
+}
+
+/* Propagate subaccesses and grp_write flags of RACC across an assignment link
+ to LACC. Enqueue sub-accesses as necessary so that the write flag is
+ propagated transitively. Return true if anything changed. Additionally, if
+ RACC is a scalar access but LACC is not, change the type of the latter, if
+ possible. */
static bool
-propagate_subaccesses_across_link (struct access *lacc, struct access *racc)
+propagate_subaccesses_from_rhs (struct access *lacc, struct access *racc)
{
struct access *rchild;
HOST_WIDE_INT norm_delta = lacc->offset - racc->offset;
bool ret = false;
+ /* IF the LHS is still not marked as being written to, we only need to do so
+ if the RHS at this level actually was. */
+ if (!lacc->grp_write)
+ {
+ gcc_checking_assert (!comes_initialized_p (racc->base));
+ if (racc->grp_write)
+ {
+ subtree_mark_written_and_rhs_enqueue (lacc);
+ ret = true;
+ }
+ }
+
if (is_gimple_reg_type (lacc->type)
|| lacc->grp_unscalarizable_region
|| racc->grp_unscalarizable_region)
- return false;
+ {
+ if (!lacc->grp_write)
+ {
+ ret = true;
+ subtree_mark_written_and_rhs_enqueue (lacc);
+ }
+ return ret;
+ }
if (is_gimple_reg_type (racc->type))
{
+ if (!lacc->grp_write)
+ {
+ ret = true;
+ subtree_mark_written_and_rhs_enqueue (lacc);
+ }
if (!lacc->first_child && !racc->first_child)
{
tree t = lacc->base;
lacc->type = racc->type;
if (build_user_friendly_ref_for_offset (&t, TREE_TYPE (t),
lacc->offset, racc->type))
- lacc->expr = t;
+ {
+ lacc->expr = t;
+ lacc->grp_same_access_path = true;
+ }
else
{
lacc->expr = build_ref_for_model (EXPR_LOCATION (lacc->base),
lacc->base, lacc->offset,
racc, NULL, false);
lacc->grp_no_warning = true;
+ lacc->grp_same_access_path = false;
}
}
- return false;
+ return ret;
}
for (rchild = racc->first_child; rchild; rchild = rchild->next_sibling)
struct access *new_acc = NULL;
HOST_WIDE_INT norm_offset = rchild->offset + norm_delta;
- if (rchild->grp_unscalarizable_region)
- continue;
-
- if (child_would_conflict_in_lacc (lacc, norm_offset, rchild->size,
+ if (child_would_conflict_in_acc (lacc, norm_offset, rchild->size,
&new_acc))
{
if (new_acc)
{
+ if (!new_acc->grp_write && rchild->grp_write)
+ {
+ gcc_assert (!lacc->grp_write);
+ subtree_mark_written_and_rhs_enqueue (new_acc);
+ ret = true;
+ }
+
rchild->grp_hint = 1;
new_acc->grp_hint |= new_acc->grp_read;
- if (rchild->first_child)
- ret |= propagate_subaccesses_across_link (new_acc, rchild);
+ if (rchild->first_child
+ && propagate_subaccesses_from_rhs (new_acc, rchild))
+ {
+ ret = 1;
+ add_access_to_rhs_work_queue (new_acc);
+ }
+ }
+ else
+ {
+ if (!lacc->grp_write)
+ {
+ ret = true;
+ subtree_mark_written_and_rhs_enqueue (lacc);
+ }
}
continue;
}
- rchild->grp_hint = 1;
- new_acc = create_artificial_child_access (lacc, rchild, norm_offset);
- if (new_acc)
+ if (rchild->grp_unscalarizable_region
+ || !budget_for_propagation_access (lacc->base))
{
- ret = true;
- if (racc->first_child)
- propagate_subaccesses_across_link (new_acc, rchild);
+ if (rchild->grp_write && !lacc->grp_write)
+ {
+ ret = true;
+ subtree_mark_written_and_rhs_enqueue (lacc);
+ }
+ continue;
}
+
+ rchild->grp_hint = 1;
+ /* Because get_ref_base_and_extent always includes padding in size for
+ accesses to DECLs but not necessarily for COMPONENT_REFs of the same
+ type, we might be actually attempting to here to create a child of the
+ same type as the parent. */
+ if (!types_compatible_p (lacc->type, rchild->type))
+ new_acc = create_artificial_child_access (lacc, rchild, norm_offset,
+ false,
+ (lacc->grp_write
+ || rchild->grp_write));
+ else
+ new_acc = lacc;
+ gcc_checking_assert (new_acc);
+ if (racc->first_child)
+ propagate_subaccesses_from_rhs (new_acc, rchild);
+
+ add_access_to_rhs_work_queue (lacc);
+ ret = true;
}
return ret;
}
+/* Propagate subaccesses of LACC across an assignment link to RACC if they
+ should inhibit total scalarization of the corresponding area. No flags are
+ being propagated in the process. Return true if anything changed. */
+
+static bool
+propagate_subaccesses_from_lhs (struct access *lacc, struct access *racc)
+{
+ if (is_gimple_reg_type (racc->type)
+ || lacc->grp_unscalarizable_region
+ || racc->grp_unscalarizable_region)
+ return false;
+
+ /* TODO: Do we want set some new racc flag to stop potential total
+ scalarization if lacc is a scalar access (and none fo the two have
+ children)? */
+
+ bool ret = false;
+ HOST_WIDE_INT norm_delta = racc->offset - lacc->offset;
+ for (struct access *lchild = lacc->first_child;
+ lchild;
+ lchild = lchild->next_sibling)
+ {
+ struct access *matching_acc = NULL;
+ HOST_WIDE_INT norm_offset = lchild->offset + norm_delta;
+
+ if (lchild->grp_unscalarizable_region
+ || child_would_conflict_in_acc (racc, norm_offset, lchild->size,
+ &matching_acc)
+ || !budget_for_propagation_access (racc->base))
+ {
+ if (matching_acc
+ && propagate_subaccesses_from_lhs (lchild, matching_acc))
+ add_access_to_lhs_work_queue (matching_acc);
+ continue;
+ }
+
+ /* Because get_ref_base_and_extent always includes padding in size for
+ accesses to DECLs but not necessarily for COMPONENT_REFs of the same
+ type, we might be actually attempting to here to create a child of the
+ same type as the parent. */
+ if (!types_compatible_p (racc->type, lchild->type))
+ {
+ struct access *new_acc
+ = create_artificial_child_access (racc, lchild, norm_offset,
+ true, false);
+ propagate_subaccesses_from_lhs (lchild, new_acc);
+ }
+ else
+ propagate_subaccesses_from_lhs (lchild, racc);
+ ret = true;
+ }
+ return ret;
+}
+
/* Propagate all subaccesses across assignment links. */
static void
propagate_all_subaccesses (void)
{
- while (work_queue_head)
+ propagation_budget = new hash_map<tree, unsigned>;
+ while (rhs_work_queue_head)
{
- struct access *racc = pop_access_from_work_queue ();
+ struct access *racc = pop_access_from_rhs_work_queue ();
struct assign_link *link;
- gcc_assert (racc->first_link);
+ if (racc->group_representative)
+ racc= racc->group_representative;
+ gcc_assert (racc->first_rhs_link);
- for (link = racc->first_link; link; link = link->next)
+ for (link = racc->first_rhs_link; link; link = link->next_rhs)
{
struct access *lacc = link->lacc;
if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
continue;
lacc = lacc->group_representative;
- if (propagate_subaccesses_across_link (lacc, racc)
- && lacc->first_link)
- add_access_to_work_queue (lacc);
+
+ bool reque_parents = false;
+ if (!bitmap_bit_p (candidate_bitmap, DECL_UID (racc->base)))
+ {
+ if (!lacc->grp_write)
+ {
+ subtree_mark_written_and_rhs_enqueue (lacc);
+ reque_parents = true;
+ }
+ }
+ else if (propagate_subaccesses_from_rhs (lacc, racc))
+ reque_parents = true;
+
+ if (reque_parents)
+ do
+ {
+ add_access_to_rhs_work_queue (lacc);
+ lacc = lacc->parent;
+ }
+ while (lacc);
}
}
-}
-
-/* Go through all accesses collected throughout the (intraprocedural) analysis
- stage, exclude overlapping ones, identify representatives and build trees
- out of them, making decisions about scalarization on the way. Return true
- iff there are any to-be-scalarized variables after this stage. */
-static bool
-analyze_all_variable_accesses (void)
-{
- int res = 0;
- bitmap tmp = BITMAP_ALLOC (NULL);
- bitmap_iterator bi;
- unsigned i;
- bool optimize_speed_p = !optimize_function_for_size_p (cfun);
+ while (lhs_work_queue_head)
+ {
+ struct access *lacc = pop_access_from_lhs_work_queue ();
+ struct assign_link *link;
- enum compiler_param param = optimize_speed_p
- ? PARAM_SRA_MAX_SCALARIZATION_SIZE_SPEED
- : PARAM_SRA_MAX_SCALARIZATION_SIZE_SIZE;
+ if (lacc->group_representative)
+ lacc = lacc->group_representative;
+ gcc_assert (lacc->first_lhs_link);
- /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
- fall back to a target default. */
- unsigned HOST_WIDE_INT max_scalarization_size
- = global_options_set.x_param_values[param]
- ? PARAM_VALUE (param)
- : get_move_ratio (optimize_speed_p) * UNITS_PER_WORD;
+ if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
+ continue;
- max_scalarization_size *= BITS_PER_UNIT;
+ for (link = lacc->first_lhs_link; link; link = link->next_lhs)
+ {
+ struct access *racc = link->racc;
- EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
- if (bitmap_bit_p (should_scalarize_away_bitmap, i)
- && !bitmap_bit_p (cannot_scalarize_away_bitmap, i))
- {
- tree var = candidate (i);
+ if (racc->group_representative)
+ racc = racc->group_representative;
+ if (!bitmap_bit_p (candidate_bitmap, DECL_UID (racc->base)))
+ continue;
+ if (propagate_subaccesses_from_lhs (lacc, racc))
+ add_access_to_lhs_work_queue (racc);
+ }
+ }
+ delete propagation_budget;
+}
- if (TREE_CODE (var) == VAR_DECL
- && type_consists_of_records_p (TREE_TYPE (var)))
- {
- if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var)))
- <= max_scalarization_size)
- {
- completely_scalarize_var (var);
- if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Will attempt to totally scalarize ");
- print_generic_expr (dump_file, var, 0);
- fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
- }
- }
- else if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Too big to totally scalarize: ");
- print_generic_expr (dump_file, var, 0);
- fprintf (dump_file, " (UID: %u)\n", DECL_UID (var));
- }
- }
- }
+/* Return true if the forest beginning with ROOT does not contain
+ unscalarizable regions or non-byte aligned accesses. */
- bitmap_copy (tmp, candidate_bitmap);
- EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
+static bool
+can_totally_scalarize_forest_p (struct access *root)
+{
+ struct access *access = root;
+ do
{
- tree var = candidate (i);
- struct access *access;
+ if (access->grp_unscalarizable_region
+ || (access->offset % BITS_PER_UNIT) != 0
+ || (access->size % BITS_PER_UNIT) != 0
+ || (is_gimple_reg_type (access->type)
+ && access->first_child))
+ return false;
- access = sort_and_splice_var_accesses (var);
- if (!access || !build_access_trees (access))
- disqualify_candidate (var,
- "No or inhibitingly overlapping accesses.");
+ if (access->first_child)
+ access = access->first_child;
+ else if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ while (access->parent && !access->next_sibling)
+ access = access->parent;
+ if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ gcc_assert (access == root);
+ root = root->next_grp;
+ access = root;
+ }
+ }
}
+ while (access);
+ return true;
+}
- propagate_all_subaccesses ();
+/* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
+ reference EXPR for total scalarization purposes and mark it as such. Within
+ the children of PARENT, link it in between PTR and NEXT_SIBLING. */
- bitmap_copy (tmp, candidate_bitmap);
+static struct access *
+create_total_scalarization_access (struct access *parent, HOST_WIDE_INT pos,
+ HOST_WIDE_INT size, tree type, tree expr,
+ struct access **ptr,
+ struct access *next_sibling)
+{
+ struct access *access = access_pool.allocate ();
+ memset (access, 0, sizeof (struct access));
+ access->base = parent->base;
+ access->offset = pos;
+ access->size = size;
+ access->expr = expr;
+ access->type = type;
+ access->parent = parent;
+ access->grp_write = parent->grp_write;
+ access->grp_total_scalarization = 1;
+ access->grp_hint = 1;
+ access->grp_same_access_path = path_comparable_for_same_access (expr);
+ access->reverse = reverse_storage_order_for_component_p (expr);
+
+ access->next_sibling = next_sibling;
+ *ptr = access;
+ return access;
+}
+
+/* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
+ reference EXPR for total scalarization purposes and mark it as such, link it
+ at *PTR and reshape the tree so that those elements at *PTR and their
+ siblings which fall within the part described by POS and SIZE are moved to
+ be children of the new access. If a partial overlap is detected, return
+ NULL. */
+
+static struct access *
+create_total_access_and_reshape (struct access *parent, HOST_WIDE_INT pos,
+ HOST_WIDE_INT size, tree type, tree expr,
+ struct access **ptr)
+{
+ struct access **p = ptr;
+
+ while (*p && (*p)->offset < pos + size)
+ {
+ if ((*p)->offset + (*p)->size > pos + size)
+ return NULL;
+ p = &(*p)->next_sibling;
+ }
+
+ struct access *next_child = *ptr;
+ struct access *new_acc
+ = create_total_scalarization_access (parent, pos, size, type, expr,
+ ptr, *p);
+ if (p != ptr)
+ {
+ new_acc->first_child = next_child;
+ *p = NULL;
+ for (struct access *a = next_child; a; a = a->next_sibling)
+ a->parent = new_acc;
+ }
+ return new_acc;
+}
+
+static bool totally_scalarize_subtree (struct access *root);
+
+/* Return true if INNER is either the same type as OUTER or if it is the type
+ of a record field in OUTER at offset zero, possibly in nested
+ sub-records. */
+
+static bool
+access_and_field_type_match_p (tree outer, tree inner)
+{
+ if (TYPE_MAIN_VARIANT (outer) == TYPE_MAIN_VARIANT (inner))
+ return true;
+ if (TREE_CODE (outer) != RECORD_TYPE)
+ return false;
+ tree fld = TYPE_FIELDS (outer);
+ while (fld)
+ {
+ if (TREE_CODE (fld) == FIELD_DECL)
+ {
+ if (!zerop (DECL_FIELD_OFFSET (fld)))
+ return false;
+ if (TYPE_MAIN_VARIANT (TREE_TYPE (fld)) == inner)
+ return true;
+ if (TREE_CODE (TREE_TYPE (fld)) == RECORD_TYPE)
+ fld = TYPE_FIELDS (TREE_TYPE (fld));
+ else
+ return false;
+ }
+ else
+ fld = DECL_CHAIN (fld);
+ }
+ return false;
+}
+
+/* Return type of total_should_skip_creating_access indicating whether a total
+ scalarization access for a field/element should be created, whether it
+ already exists or whether the entire total scalarization has to fail. */
+
+enum total_sra_field_state {TOTAL_FLD_CREATE, TOTAL_FLD_DONE, TOTAL_FLD_FAILED};
+
+/* Do all the necessary steps in total scalarization when the given aggregate
+ type has a TYPE at POS with the given SIZE should be put into PARENT and
+ when we have processed all its siblings with smaller offsets up until and
+ including LAST_SEEN_SIBLING (which can be NULL).
+
+ If some further siblings are to be skipped, set *LAST_SEEN_SIBLING as
+ appropriate. Return TOTAL_FLD_CREATE id the caller should carry on with
+ creating a new access, TOTAL_FLD_DONE if access or accesses capable of
+ representing the described part of the aggregate for the purposes of total
+ scalarization already exist or TOTAL_FLD_FAILED if there is a problem which
+ prevents total scalarization from happening at all. */
+
+static enum total_sra_field_state
+total_should_skip_creating_access (struct access *parent,
+ struct access **last_seen_sibling,
+ tree type, HOST_WIDE_INT pos,
+ HOST_WIDE_INT size)
+{
+ struct access *next_child;
+ if (!*last_seen_sibling)
+ next_child = parent->first_child;
+ else
+ next_child = (*last_seen_sibling)->next_sibling;
+
+ /* First, traverse the chain of siblings until it points to an access with
+ offset at least equal to POS. Check all skipped accesses whether they
+ span the POS boundary and if so, return with a failure. */
+ while (next_child && next_child->offset < pos)
+ {
+ if (next_child->offset + next_child->size > pos)
+ return TOTAL_FLD_FAILED;
+ *last_seen_sibling = next_child;
+ next_child = next_child->next_sibling;
+ }
+
+ /* Now check whether next_child has exactly the right POS and SIZE and if so,
+ whether it can represent what we need and can be totally scalarized
+ itself. */
+ if (next_child && next_child->offset == pos
+ && next_child->size == size)
+ {
+ if (!is_gimple_reg_type (next_child->type)
+ && (!access_and_field_type_match_p (type, next_child->type)
+ || !totally_scalarize_subtree (next_child)))
+ return TOTAL_FLD_FAILED;
+
+ *last_seen_sibling = next_child;
+ return TOTAL_FLD_DONE;
+ }
+
+ /* If the child we're looking at would partially overlap, we just cannot
+ totally scalarize. */
+ if (next_child
+ && next_child->offset < pos + size
+ && next_child->offset + next_child->size > pos + size)
+ return TOTAL_FLD_FAILED;
+
+ if (is_gimple_reg_type (type))
+ {
+ /* We don't scalarize accesses that are children of other scalar type
+ accesses, so if we go on and create an access for a register type,
+ there should not be any pre-existing children. There are rare cases
+ where the requested type is a vector but we already have register
+ accesses for all its elements which is equally good. Detect that
+ situation or whether we need to bail out. */
+
+ HOST_WIDE_INT covered = pos;
+ bool skipping = false;
+ while (next_child
+ && next_child->offset + next_child->size <= pos + size)
+ {
+ if (next_child->offset != covered
+ || !is_gimple_reg_type (next_child->type))
+ return TOTAL_FLD_FAILED;
+
+ covered += next_child->size;
+ *last_seen_sibling = next_child;
+ next_child = next_child->next_sibling;
+ skipping = true;
+ }
+
+ if (skipping)
+ {
+ if (covered != pos + size)
+ return TOTAL_FLD_FAILED;
+ else
+ return TOTAL_FLD_DONE;
+ }
+ }
+
+ return TOTAL_FLD_CREATE;
+}
+
+/* Go over sub-tree rooted in ROOT and attempt to create scalar accesses
+ spanning all uncovered areas covered by ROOT, return false if the attempt
+ failed. All created accesses will have grp_unscalarizable_region set (and
+ should be ignored if the function returns false). */
+
+static bool
+totally_scalarize_subtree (struct access *root)
+{
+ gcc_checking_assert (!root->grp_unscalarizable_region);
+ gcc_checking_assert (!is_gimple_reg_type (root->type));
+
+ struct access *last_seen_sibling = NULL;
+
+ switch (TREE_CODE (root->type))
+ {
+ case RECORD_TYPE:
+ for (tree fld = TYPE_FIELDS (root->type); fld; fld = DECL_CHAIN (fld))
+ if (TREE_CODE (fld) == FIELD_DECL)
+ {
+ tree ft = TREE_TYPE (fld);
+ HOST_WIDE_INT fsize = tree_to_uhwi (DECL_SIZE (fld));
+ if (!fsize)
+ continue;
+
+ HOST_WIDE_INT pos = root->offset + int_bit_position (fld);
+ enum total_sra_field_state
+ state = total_should_skip_creating_access (root,
+ &last_seen_sibling,
+ ft, pos, fsize);
+ switch (state)
+ {
+ case TOTAL_FLD_FAILED:
+ return false;
+ case TOTAL_FLD_DONE:
+ continue;
+ case TOTAL_FLD_CREATE:
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ struct access **p = (last_seen_sibling
+ ? &last_seen_sibling->next_sibling
+ : &root->first_child);
+ tree nref = build3 (COMPONENT_REF, ft, root->expr, fld, NULL_TREE);
+ struct access *new_child
+ = create_total_access_and_reshape (root, pos, fsize, ft, nref, p);
+ if (!new_child)
+ return false;
+
+ if (!is_gimple_reg_type (ft)
+ && !totally_scalarize_subtree (new_child))
+ return false;
+ last_seen_sibling = new_child;
+ }
+ break;
+ case ARRAY_TYPE:
+ {
+ tree elemtype = TREE_TYPE (root->type);
+ tree elem_size = TYPE_SIZE (elemtype);
+ gcc_assert (elem_size && tree_fits_shwi_p (elem_size));
+ HOST_WIDE_INT el_size = tree_to_shwi (elem_size);
+ gcc_assert (el_size > 0);
+
+ tree minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (root->type));
+ gcc_assert (TREE_CODE (minidx) == INTEGER_CST);
+ tree maxidx = TYPE_MAX_VALUE (TYPE_DOMAIN (root->type));
+ /* Skip (some) zero-length arrays; others have MAXIDX == MINIDX - 1. */
+ if (!maxidx)
+ goto out;
+ gcc_assert (TREE_CODE (maxidx) == INTEGER_CST);
+ tree domain = TYPE_DOMAIN (root->type);
+ /* MINIDX and MAXIDX are inclusive, and must be interpreted in
+ DOMAIN (e.g. signed int, whereas min/max may be size_int). */
+ offset_int idx = wi::to_offset (minidx);
+ offset_int max = wi::to_offset (maxidx);
+ if (!TYPE_UNSIGNED (domain))
+ {
+ idx = wi::sext (idx, TYPE_PRECISION (domain));
+ max = wi::sext (max, TYPE_PRECISION (domain));
+ }
+ for (HOST_WIDE_INT pos = root->offset;
+ idx <= max;
+ pos += el_size, ++idx)
+ {
+ enum total_sra_field_state
+ state = total_should_skip_creating_access (root,
+ &last_seen_sibling,
+ elemtype, pos,
+ el_size);
+ switch (state)
+ {
+ case TOTAL_FLD_FAILED:
+ return false;
+ case TOTAL_FLD_DONE:
+ continue;
+ case TOTAL_FLD_CREATE:
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ struct access **p = (last_seen_sibling
+ ? &last_seen_sibling->next_sibling
+ : &root->first_child);
+ tree nref = build4 (ARRAY_REF, elemtype, root->expr,
+ wide_int_to_tree (domain, idx),
+ NULL_TREE, NULL_TREE);
+ struct access *new_child
+ = create_total_access_and_reshape (root, pos, el_size, elemtype,
+ nref, p);
+ if (!new_child)
+ return false;
+
+ if (!is_gimple_reg_type (elemtype)
+ && !totally_scalarize_subtree (new_child))
+ return false;
+ last_seen_sibling = new_child;
+ }
+ }
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ out:
+ return true;
+}
+
+/* Go through all accesses collected throughout the (intraprocedural) analysis
+ stage, exclude overlapping ones, identify representatives and build trees
+ out of them, making decisions about scalarization on the way. Return true
+ iff there are any to-be-scalarized variables after this stage. */
+
+static bool
+analyze_all_variable_accesses (void)
+{
+ int res = 0;
+ bitmap tmp = BITMAP_ALLOC (NULL);
+ bitmap_iterator bi;
+ unsigned i;
+
+ bitmap_copy (tmp, candidate_bitmap);
+ EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
+ {
+ tree var = candidate (i);
+ struct access *access;
+
+ access = sort_and_splice_var_accesses (var);
+ if (!access || !build_access_trees (access))
+ disqualify_candidate (var,
+ "No or inhibitingly overlapping accesses.");
+ }
+
+ propagate_all_subaccesses ();
+
+ bool optimize_speed_p = !optimize_function_for_size_p (cfun);
+ /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
+ fall back to a target default. */
+ unsigned HOST_WIDE_INT max_scalarization_size
+ = get_move_ratio (optimize_speed_p) * UNITS_PER_WORD;
+
+ if (optimize_speed_p)
+ {
+ if (global_options_set.x_param_sra_max_scalarization_size_speed)
+ max_scalarization_size = param_sra_max_scalarization_size_speed;
+ }
+ else
+ {
+ if (global_options_set.x_param_sra_max_scalarization_size_size)
+ max_scalarization_size = param_sra_max_scalarization_size_size;
+ }
+ max_scalarization_size *= BITS_PER_UNIT;
+
+ EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
+ if (bitmap_bit_p (should_scalarize_away_bitmap, i)
+ && !bitmap_bit_p (cannot_scalarize_away_bitmap, i))
+ {
+ tree var = candidate (i);
+ if (!VAR_P (var))
+ continue;
+
+ if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))) > max_scalarization_size)
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Too big to totally scalarize: ");
+ print_generic_expr (dump_file, var);
+ fprintf (dump_file, " (UID: %u)\n", DECL_UID (var));
+ }
+ continue;
+ }
+
+ bool all_types_ok = true;
+ for (struct access *access = get_first_repr_for_decl (var);
+ access;
+ access = access->next_grp)
+ if (!can_totally_scalarize_forest_p (access)
+ || !scalarizable_type_p (access->type, constant_decl_p (var)))
+ {
+ all_types_ok = false;
+ break;
+ }
+ if (!all_types_ok)
+ continue;
+
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Will attempt to totally scalarize ");
+ print_generic_expr (dump_file, var);
+ fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
+ }
+ bool scalarized = true;
+ for (struct access *access = get_first_repr_for_decl (var);
+ access;
+ access = access->next_grp)
+ if (!is_gimple_reg_type (access->type)
+ && !totally_scalarize_subtree (access))
+ {
+ scalarized = false;
+ break;
+ }
+
+ if (scalarized)
+ for (struct access *access = get_first_repr_for_decl (var);
+ access;
+ access = access->next_grp)
+ access->grp_total_scalarization = true;
+ }
+
+ if (flag_checking)
+ verify_all_sra_access_forests ();
+
+ bitmap_copy (tmp, candidate_bitmap);
EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
{
tree var = candidate (i);
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file, "\nAccess trees for ");
- print_generic_expr (dump_file, var, 0);
+ print_generic_expr (dump_file, var);
fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
dump_access_tree (dump_file, access);
fprintf (dump_file, "\n");
gimple_stmt_iterator *gsi, bool write,
bool insert_after, location_t loc)
{
+ /* Never write anything into constant pool decls. See PR70602. */
+ if (!write && constant_decl_p (agg))
+ return;
do
{
if (chunk_size && access->offset >= start_offset + chunk_size)
}
/* Assign zero to all scalar replacements in an access subtree. ACCESS is the
- the root of the subtree to be processed. GSI is the statement iterator used
+ root of the subtree to be processed. GSI is the statement iterator used
for inserting statements which are added after the current statement if
INSERT_AFTER is true or before it otherwise. */
init_subtree_with_zero (child, gsi, insert_after, loc);
}
-/* Clobber all scalar replacements in an access subtree. ACCESS is the the
+/* Clobber all scalar replacements in an access subtree. ACCESS is the
root of the subtree to be processed. GSI is the statement iterator used
for inserting statements which are added after the current statement if
INSERT_AFTER is true or before it otherwise. */
if (access->grp_to_be_replaced)
{
tree rep = get_access_replacement (access);
- tree clobber = build_constructor (access->type, NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
- gimple stmt = gimple_build_assign (rep, clobber);
+ tree clobber = build_clobber (access->type);
+ gimple *stmt = gimple_build_assign (rep, clobber);
if (insert_after)
gsi_insert_after (gsi, stmt, GSI_NEW_STMT);
static struct access *
get_access_for_expr (tree expr)
{
- HOST_WIDE_INT offset, size, max_size;
+ poly_int64 poffset, psize, pmax_size;
+ HOST_WIDE_INT offset, max_size;
tree base;
+ bool reverse;
/* FIXME: This should not be necessary but Ada produces V_C_Es with a type of
a different size than the size of its argument and we need the latter
if (TREE_CODE (expr) == VIEW_CONVERT_EXPR)
expr = TREE_OPERAND (expr, 0);
- base = get_ref_base_and_extent (expr, &offset, &size, &max_size);
- if (max_size == -1 || !DECL_P (base))
+ base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size,
+ &reverse);
+ if (!known_size_p (pmax_size)
+ || !pmax_size.is_constant (&max_size)
+ || !poffset.is_constant (&offset)
+ || !DECL_P (base))
return NULL;
- if (!bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
+ if (tree basesize = DECL_SIZE (base))
+ {
+ poly_int64 sz;
+ if (offset < 0
+ || !poly_int_tree_p (basesize, &sz)
+ || known_le (sz, offset))
+ return NULL;
+ }
+
+ if (max_size == 0
+ || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
return NULL;
return get_var_base_offset_size_access (base, offset, max_size);
location_t loc;
struct access *access;
tree type, bfr, orig_expr;
+ bool partial_cplx_access = false;
if (TREE_CODE (*expr) == BIT_FIELD_REF)
{
bfr = NULL_TREE;
if (TREE_CODE (*expr) == REALPART_EXPR || TREE_CODE (*expr) == IMAGPART_EXPR)
- expr = &TREE_OPERAND (*expr, 0);
+ {
+ expr = &TREE_OPERAND (*expr, 0);
+ partial_cplx_access = true;
+ }
access = get_access_for_expr (*expr);
if (!access)
return false;
be accessed as a different type too, potentially creating a need for
type conversion (see PR42196) and when scalarized unions are involved
in assembler statements (see PR42398). */
- if (!useless_type_conversion_p (type, access->type))
+ if (!bfr && !useless_type_conversion_p (type, access->type))
{
tree ref;
ref = build_ref_for_model (loc, orig_expr, 0, access, gsi, false);
- if (write)
+ if (partial_cplx_access)
+ {
+ /* VIEW_CONVERT_EXPRs in partial complex access are always fine in
+ the case of a write because in such case the replacement cannot
+ be a gimple register. In the case of a load, we have to
+ differentiate in between a register an non-register
+ replacement. */
+ tree t = build1 (VIEW_CONVERT_EXPR, type, repl);
+ gcc_checking_assert (!write || access->grp_partial_lhs);
+ if (!access->grp_partial_lhs)
+ {
+ tree tmp = make_ssa_name (type);
+ gassign *stmt = gimple_build_assign (tmp, t);
+ /* This is always a read. */
+ gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
+ t = tmp;
+ }
+ *expr = t;
+ }
+ else if (write)
{
gassign *stmt;
if (racc && racc->grp_to_be_replaced)
{
- if (racc->grp_write)
+ if (racc->grp_write || constant_decl_p (racc->base))
drhs = get_access_replacement (racc);
else
drhs = NULL;
the same values as sra_modify_assign. */
static enum assignment_mod_result
-sra_modify_constructor_assign (gimple stmt, gimple_stmt_iterator *gsi)
+sra_modify_constructor_assign (gimple *stmt, gimple_stmt_iterator *gsi)
{
tree lhs = gimple_assign_lhs (stmt);
struct access *acc = get_access_for_expr (lhs);
return SRA_AM_MODIFIED;
}
- if (vec_safe_length (CONSTRUCTOR_ELTS (gimple_assign_rhs1 (stmt))) > 0)
+ if (CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt)) > 0)
{
/* I have never seen this code path trigger but if it can happen the
following should handle it gracefully. */
/* Create and return a new suitable default definition SSA_NAME for RACC which
is an access describing an uninitialized part of an aggregate that is being
- loaded. */
+ loaded. REG_TREE is used instead of the actual RACC type if that is not of
+ a gimple register type. */
static tree
-get_repl_default_def_ssa_name (struct access *racc)
+get_repl_default_def_ssa_name (struct access *racc, tree reg_type)
{
gcc_checking_assert (!racc->grp_to_be_replaced
&& !racc->grp_to_be_debug_replaced);
if (!racc->replacement_decl)
- racc->replacement_decl = create_access_replacement (racc);
+ racc->replacement_decl = create_access_replacement (racc, reg_type);
return get_or_create_ssa_default_def (cfun, racc->replacement_decl);
}
-/* Return true if REF has an VIEW_CONVERT_EXPR or a COMPONENT_REF with a
- bit-field field declaration somewhere in it. */
-
-static inline bool
-contains_vce_or_bfcref_p (const_tree ref)
-{
- while (handled_component_p (ref))
- {
- if (TREE_CODE (ref) == VIEW_CONVERT_EXPR
- || (TREE_CODE (ref) == COMPONENT_REF
- && DECL_BIT_FIELD (TREE_OPERAND (ref, 1))))
- return true;
- ref = TREE_OPERAND (ref, 0);
- }
-
- return false;
-}
-
/* Examine both sides of the assignment statement pointed to by STMT, replace
them with a scalare replacement if there is one and generate copying of
replacements if scalarized aggregates have been used in the assignment. GSI
copying. */
static enum assignment_mod_result
-sra_modify_assign (gimple stmt, gimple_stmt_iterator *gsi)
+sra_modify_assign (gimple *stmt, gimple_stmt_iterator *gsi)
{
struct access *lacc, *racc;
tree lhs, rhs;
racc = get_access_for_expr (rhs);
if (!lacc && !racc)
return SRA_AM_NONE;
+ /* Avoid modifying initializations of constant-pool replacements. */
+ if (racc && (racc->replacement_decl == lhs))
+ return SRA_AM_NONE;
loc = gimple_location (stmt);
if (lacc && lacc->grp_to_be_replaced)
}
else if (racc
&& !racc->grp_unscalarized_data
+ && !racc->grp_unscalarizable_region
&& TREE_CODE (lhs) == SSA_NAME
&& !access_has_replacements_p (racc))
{
- rhs = get_repl_default_def_ssa_name (racc);
+ rhs = get_repl_default_def_ssa_name (racc, TREE_TYPE (lhs));
modify_this_stmt = true;
sra_stats.exprs++;
}
lhs = build_ref_for_model (loc, lhs, 0, racc, gsi, false);
gimple_assign_set_lhs (stmt, lhs);
}
- else if (AGGREGATE_TYPE_P (TREE_TYPE (rhs))
+ else if (lacc
+ && AGGREGATE_TYPE_P (TREE_TYPE (rhs))
&& !contains_vce_or_bfcref_p (rhs))
rhs = build_ref_for_model (loc, rhs, 0, lacc, gsi, false);
|| contains_vce_or_bfcref_p (lhs)
|| stmt_ends_bb_p (stmt))
{
- if (access_has_children_p (racc))
+ /* No need to copy into a constant-pool, it comes pre-initialized. */
+ if (access_has_children_p (racc) && !constant_decl_p (racc->base))
generate_subtree_copies (racc->first_child, rhs, racc->offset, 0, 0,
gsi, false, false, loc);
if (access_has_children_p (lacc))
else
{
if (access_has_children_p (racc)
- && !racc->grp_unscalarized_data)
+ && !racc->grp_unscalarized_data
+ && TREE_CODE (lhs) != SSA_NAME)
{
if (dump_file)
{
fprintf (dump_file, "Removing load: ");
- print_gimple_stmt (dump_file, stmt, 0, 0);
+ print_gimple_stmt (dump_file, stmt, 0);
}
generate_subtree_copies (racc->first_child, lhs,
racc->offset, 0, 0, gsi,
}
}
+/* Set any scalar replacements of values in the constant pool to the initial
+ value of the constant. (Constant-pool decls like *.LC0 have effectively
+ been initialized before the program starts, we must do the same for their
+ replacements.) Thus, we output statements like 'SR.1 = *.LC0[0];' into
+ the function's entry block. */
+
+static void
+initialize_constant_pool_replacements (void)
+{
+ gimple_seq seq = NULL;
+ gimple_stmt_iterator gsi = gsi_start (seq);
+ bitmap_iterator bi;
+ unsigned i;
+
+ EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
+ {
+ tree var = candidate (i);
+ if (!constant_decl_p (var))
+ continue;
+
+ struct access *access = get_first_repr_for_decl (var);
+
+ while (access)
+ {
+ if (access->replacement_decl)
+ {
+ gassign *stmt
+ = gimple_build_assign (get_access_replacement (access),
+ unshare_expr (access->expr));
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Generating constant initializer: ");
+ print_gimple_stmt (dump_file, stmt, 0);
+ fprintf (dump_file, "\n");
+ }
+ gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
+ update_stmt (stmt);
+ }
+
+ if (access->first_child)
+ access = access->first_child;
+ else if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ while (access->parent && !access->next_sibling)
+ access = access->parent;
+ if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ access = access->next_grp;
+ }
+ }
+ }
+
+ seq = gsi_seq (gsi);
+ if (seq)
+ gsi_insert_seq_on_edge_immediate (
+ single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
+}
+
/* Traverse the function body and all modifications as decided in
analyze_all_variable_accesses. Return true iff the CFG has been
changed. */
bool cfg_changed = false;
basic_block bb;
+ initialize_constant_pool_replacements ();
+
FOR_EACH_BB_FN (bb, cfun)
{
gimple_stmt_iterator gsi = gsi_start_bb (bb);
while (!gsi_end_p (gsi))
{
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
enum assignment_mod_result assign_result;
bool modified = false, deleted = false;
tree *t;
{
return new pass_sra (ctxt);
}
-
-
-/* Return true iff PARM (which must be a parm_decl) is an unused scalar
- parameter. */
-
-static bool
-is_unused_scalar_param (tree parm)
-{
- tree name;
- return (is_gimple_reg (parm)
- && (!(name = ssa_default_def (cfun, parm))
- || has_zero_uses (name)));
-}
-
-/* Scan immediate uses of a default definition SSA name of a parameter PARM and
- examine whether there are any direct or otherwise infeasible ones. If so,
- return true, otherwise return false. PARM must be a gimple register with a
- non-NULL default definition. */
-
-static bool
-ptr_parm_has_direct_uses (tree parm)
-{
- imm_use_iterator ui;
- gimple stmt;
- tree name = ssa_default_def (cfun, parm);
- bool ret = false;
-
- FOR_EACH_IMM_USE_STMT (stmt, ui, name)
- {
- int uses_ok = 0;
- use_operand_p use_p;
-
- if (is_gimple_debug (stmt))
- continue;
-
- /* Valid uses include dereferences on the lhs and the rhs. */
- if (gimple_has_lhs (stmt))
- {
- tree lhs = gimple_get_lhs (stmt);
- while (handled_component_p (lhs))
- lhs = TREE_OPERAND (lhs, 0);
- if (TREE_CODE (lhs) == MEM_REF
- && TREE_OPERAND (lhs, 0) == name
- && integer_zerop (TREE_OPERAND (lhs, 1))
- && types_compatible_p (TREE_TYPE (lhs),
- TREE_TYPE (TREE_TYPE (name)))
- && !TREE_THIS_VOLATILE (lhs))
- uses_ok++;
- }
- if (gimple_assign_single_p (stmt))
- {
- tree rhs = gimple_assign_rhs1 (stmt);
- while (handled_component_p (rhs))
- rhs = TREE_OPERAND (rhs, 0);
- if (TREE_CODE (rhs) == MEM_REF
- && TREE_OPERAND (rhs, 0) == name
- && integer_zerop (TREE_OPERAND (rhs, 1))
- && types_compatible_p (TREE_TYPE (rhs),
- TREE_TYPE (TREE_TYPE (name)))
- && !TREE_THIS_VOLATILE (rhs))
- uses_ok++;
- }
- else if (is_gimple_call (stmt))
- {
- unsigned i;
- for (i = 0; i < gimple_call_num_args (stmt); ++i)
- {
- tree arg = gimple_call_arg (stmt, i);
- while (handled_component_p (arg))
- arg = TREE_OPERAND (arg, 0);
- if (TREE_CODE (arg) == MEM_REF
- && TREE_OPERAND (arg, 0) == name
- && integer_zerop (TREE_OPERAND (arg, 1))
- && types_compatible_p (TREE_TYPE (arg),
- TREE_TYPE (TREE_TYPE (name)))
- && !TREE_THIS_VOLATILE (arg))
- uses_ok++;
- }
- }
-
- /* If the number of valid uses does not match the number of
- uses in this stmt there is an unhandled use. */
- FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
- --uses_ok;
-
- if (uses_ok != 0)
- ret = true;
-
- if (ret)
- BREAK_FROM_IMM_USE_STMT (ui);
- }
-
- return ret;
-}
-
-/* Identify candidates for reduction for IPA-SRA based on their type and mark
- them in candidate_bitmap. Note that these do not necessarily include
- parameter which are unused and thus can be removed. Return true iff any
- such candidate has been found. */
-
-static bool
-find_param_candidates (void)
-{
- tree parm;
- int count = 0;
- bool ret = false;
- const char *msg;
-
- for (parm = DECL_ARGUMENTS (current_function_decl);
- parm;
- parm = DECL_CHAIN (parm))
- {
- tree type = TREE_TYPE (parm);
- tree_node **slot;
-
- count++;
-
- if (TREE_THIS_VOLATILE (parm)
- || TREE_ADDRESSABLE (parm)
- || (!is_gimple_reg_type (type) && is_va_list_type (type)))
- continue;
-
- if (is_unused_scalar_param (parm))
- {
- ret = true;
- continue;
- }
-
- if (POINTER_TYPE_P (type))
- {
- type = TREE_TYPE (type);
-
- if (TREE_CODE (type) == FUNCTION_TYPE
- || TYPE_VOLATILE (type)
- || (TREE_CODE (type) == ARRAY_TYPE
- && TYPE_NONALIASED_COMPONENT (type))
- || !is_gimple_reg (parm)
- || is_va_list_type (type)
- || ptr_parm_has_direct_uses (parm))
- continue;
- }
- else if (!AGGREGATE_TYPE_P (type))
- continue;
-
- if (!COMPLETE_TYPE_P (type)
- || !tree_fits_uhwi_p (TYPE_SIZE (type))
- || tree_to_uhwi (TYPE_SIZE (type)) == 0
- || (AGGREGATE_TYPE_P (type)
- && type_internals_preclude_sra_p (type, &msg)))
- continue;
-
- bitmap_set_bit (candidate_bitmap, DECL_UID (parm));
- slot = candidates->find_slot_with_hash (parm, DECL_UID (parm), INSERT);
- *slot = parm;
-
- ret = true;
- if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Candidate (%d): ", DECL_UID (parm));
- print_generic_expr (dump_file, parm, 0);
- fprintf (dump_file, "\n");
- }
- }
-
- func_param_count = count;
- return ret;
-}
-
-/* Callback of walk_aliased_vdefs, marks the access passed as DATA as
- maybe_modified. */
-
-static bool
-mark_maybe_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
- void *data)
-{
- struct access *repr = (struct access *) data;
-
- repr->grp_maybe_modified = 1;
- return true;
-}
-
-/* Analyze what representatives (in linked lists accessible from
- REPRESENTATIVES) can be modified by side effects of statements in the
- current function. */
-
-static void
-analyze_modified_params (vec<access_p> representatives)
-{
- int i;
-
- for (i = 0; i < func_param_count; i++)
- {
- struct access *repr;
-
- for (repr = representatives[i];
- repr;
- repr = repr->next_grp)
- {
- struct access *access;
- bitmap visited;
- ao_ref ar;
-
- if (no_accesses_p (repr))
- continue;
- if (!POINTER_TYPE_P (TREE_TYPE (repr->base))
- || repr->grp_maybe_modified)
- continue;
-
- ao_ref_init (&ar, repr->expr);
- visited = BITMAP_ALLOC (NULL);
- for (access = repr; access; access = access->next_sibling)
- {
- /* All accesses are read ones, otherwise grp_maybe_modified would
- be trivially set. */
- walk_aliased_vdefs (&ar, gimple_vuse (access->stmt),
- mark_maybe_modified, repr, &visited);
- if (repr->grp_maybe_modified)
- break;
- }
- BITMAP_FREE (visited);
- }
- }
-}
-
-/* Propagate distances in bb_dereferences in the opposite direction than the
- control flow edges, in each step storing the maximum of the current value
- and the minimum of all successors. These steps are repeated until the table
- stabilizes. Note that BBs which might terminate the functions (according to
- final_bbs bitmap) never updated in this way. */
-
-static void
-propagate_dereference_distances (void)
-{
- basic_block bb;
-
- auto_vec<basic_block> queue (last_basic_block_for_fn (cfun));
- queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (cfun));
- FOR_EACH_BB_FN (bb, cfun)
- {
- queue.quick_push (bb);
- bb->aux = bb;
- }
-
- while (!queue.is_empty ())
- {
- edge_iterator ei;
- edge e;
- bool change = false;
- int i;
-
- bb = queue.pop ();
- bb->aux = NULL;
-
- if (bitmap_bit_p (final_bbs, bb->index))
- continue;
-
- for (i = 0; i < func_param_count; i++)
- {
- int idx = bb->index * func_param_count + i;
- bool first = true;
- HOST_WIDE_INT inh = 0;
-
- FOR_EACH_EDGE (e, ei, bb->succs)
- {
- int succ_idx = e->dest->index * func_param_count + i;
-
- if (e->src == EXIT_BLOCK_PTR_FOR_FN (cfun))
- continue;
-
- if (first)
- {
- first = false;
- inh = bb_dereferences [succ_idx];
- }
- else if (bb_dereferences [succ_idx] < inh)
- inh = bb_dereferences [succ_idx];
- }
-
- if (!first && bb_dereferences[idx] < inh)
- {
- bb_dereferences[idx] = inh;
- change = true;
- }
- }
-
- if (change && !bitmap_bit_p (final_bbs, bb->index))
- FOR_EACH_EDGE (e, ei, bb->preds)
- {
- if (e->src->aux)
- continue;
-
- e->src->aux = e->src;
- queue.quick_push (e->src);
- }
- }
-}
-
-/* Dump a dereferences TABLE with heading STR to file F. */
-
-static void
-dump_dereferences_table (FILE *f, const char *str, HOST_WIDE_INT *table)
-{
- basic_block bb;
-
- fprintf (dump_file, "%s", str);
- FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun),
- EXIT_BLOCK_PTR_FOR_FN (cfun), next_bb)
- {
- fprintf (f, "%4i %i ", bb->index, bitmap_bit_p (final_bbs, bb->index));
- if (bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
- {
- int i;
- for (i = 0; i < func_param_count; i++)
- {
- int idx = bb->index * func_param_count + i;
- fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", table[idx]);
- }
- }
- fprintf (f, "\n");
- }
- fprintf (dump_file, "\n");
-}
-
-/* Determine what (parts of) parameters passed by reference that are not
- assigned to are not certainly dereferenced in this function and thus the
- dereferencing cannot be safely moved to the caller without potentially
- introducing a segfault. Mark such REPRESENTATIVES as
- grp_not_necessarilly_dereferenced.
-
- The dereferenced maximum "distance," i.e. the offset + size of the accessed
- part is calculated rather than simple booleans are calculated for each
- pointer parameter to handle cases when only a fraction of the whole
- aggregate is allocated (see testsuite/gcc.c-torture/execute/ipa-sra-2.c for
- an example).
-
- The maximum dereference distances for each pointer parameter and BB are
- already stored in bb_dereference. This routine simply propagates these
- values upwards by propagate_dereference_distances and then compares the
- distances of individual parameters in the ENTRY BB to the equivalent
- distances of each representative of a (fraction of a) parameter. */
-
-static void
-analyze_caller_dereference_legality (vec<access_p> representatives)
-{
- int i;
-
- if (dump_file && (dump_flags & TDF_DETAILS))
- dump_dereferences_table (dump_file,
- "Dereference table before propagation:\n",
- bb_dereferences);
-
- propagate_dereference_distances ();
-
- if (dump_file && (dump_flags & TDF_DETAILS))
- dump_dereferences_table (dump_file,
- "Dereference table after propagation:\n",
- bb_dereferences);
-
- for (i = 0; i < func_param_count; i++)
- {
- struct access *repr = representatives[i];
- int idx = ENTRY_BLOCK_PTR_FOR_FN (cfun)->index * func_param_count + i;
-
- if (!repr || no_accesses_p (repr))
- continue;
-
- do
- {
- if ((repr->offset + repr->size) > bb_dereferences[idx])
- repr->grp_not_necessarilly_dereferenced = 1;
- repr = repr->next_grp;
- }
- while (repr);
- }
-}
-
-/* Return the representative access for the parameter declaration PARM if it is
- a scalar passed by reference which is not written to and the pointer value
- is not used directly. Thus, if it is legal to dereference it in the caller
- and we can rule out modifications through aliases, such parameter should be
- turned into one passed by value. Return NULL otherwise. */
-
-static struct access *
-unmodified_by_ref_scalar_representative (tree parm)
-{
- int i, access_count;
- struct access *repr;
- vec<access_p> *access_vec;
-
- access_vec = get_base_access_vector (parm);
- gcc_assert (access_vec);
- repr = (*access_vec)[0];
- if (repr->write)
- return NULL;
- repr->group_representative = repr;
-
- access_count = access_vec->length ();
- for (i = 1; i < access_count; i++)
- {
- struct access *access = (*access_vec)[i];
- if (access->write)
- return NULL;
- access->group_representative = repr;
- access->next_sibling = repr->next_sibling;
- repr->next_sibling = access;
- }
-
- repr->grp_read = 1;
- repr->grp_scalar_ptr = 1;
- return repr;
-}
-
-/* Return true iff this ACCESS precludes IPA-SRA of the parameter it is
- associated with. REQ_ALIGN is the minimum required alignment. */
-
-static bool
-access_precludes_ipa_sra_p (struct access *access, unsigned int req_align)
-{
- unsigned int exp_align;
- /* Avoid issues such as the second simple testcase in PR 42025. The problem
- is incompatible assign in a call statement (and possibly even in asm
- statements). This can be relaxed by using a new temporary but only for
- non-TREE_ADDRESSABLE types and is probably not worth the complexity. (In
- intraprocedural SRA we deal with this by keeping the old aggregate around,
- something we cannot do in IPA-SRA.) */
- if (access->write
- && (is_gimple_call (access->stmt)
- || gimple_code (access->stmt) == GIMPLE_ASM))
- return true;
-
- exp_align = get_object_alignment (access->expr);
- if (exp_align < req_align)
- return true;
-
- return false;
-}
-
-
-/* Sort collected accesses for parameter PARM, identify representatives for
- each accessed region and link them together. Return NULL if there are
- different but overlapping accesses, return the special ptr value meaning
- there are no accesses for this parameter if that is the case and return the
- first representative otherwise. Set *RO_GRP if there is a group of accesses
- with only read (i.e. no write) accesses. */
-
-static struct access *
-splice_param_accesses (tree parm, bool *ro_grp)
-{
- int i, j, access_count, group_count;
- int agg_size, total_size = 0;
- struct access *access, *res, **prev_acc_ptr = &res;
- vec<access_p> *access_vec;
-
- access_vec = get_base_access_vector (parm);
- if (!access_vec)
- return &no_accesses_representant;
- access_count = access_vec->length ();
-
- access_vec->qsort (compare_access_positions);
-
- i = 0;
- total_size = 0;
- group_count = 0;
- while (i < access_count)
- {
- bool modification;
- tree a1_alias_type;
- access = (*access_vec)[i];
- modification = access->write;
- if (access_precludes_ipa_sra_p (access, TYPE_ALIGN (access->type)))
- return NULL;
- a1_alias_type = reference_alias_ptr_type (access->expr);
-
- /* Access is about to become group representative unless we find some
- nasty overlap which would preclude us from breaking this parameter
- apart. */
-
- j = i + 1;
- while (j < access_count)
- {
- struct access *ac2 = (*access_vec)[j];
- if (ac2->offset != access->offset)
- {
- /* All or nothing law for parameters. */
- if (access->offset + access->size > ac2->offset)
- return NULL;
- else
- break;
- }
- else if (ac2->size != access->size)
- return NULL;
-
- if (access_precludes_ipa_sra_p (ac2, TYPE_ALIGN (access->type))
- || (ac2->type != access->type
- && (TREE_ADDRESSABLE (ac2->type)
- || TREE_ADDRESSABLE (access->type)))
- || (reference_alias_ptr_type (ac2->expr) != a1_alias_type))
- return NULL;
-
- modification |= ac2->write;
- ac2->group_representative = access;
- ac2->next_sibling = access->next_sibling;
- access->next_sibling = ac2;
- j++;
- }
-
- group_count++;
- access->grp_maybe_modified = modification;
- if (!modification)
- *ro_grp = true;
- *prev_acc_ptr = access;
- prev_acc_ptr = &access->next_grp;
- total_size += access->size;
- i = j;
- }
-
- if (POINTER_TYPE_P (TREE_TYPE (parm)))
- agg_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (parm))));
- else
- agg_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
- if (total_size >= agg_size)
- return NULL;
-
- gcc_assert (group_count > 0);
- return res;
-}
-
-/* Decide whether parameters with representative accesses given by REPR should
- be reduced into components. */
-
-static int
-decide_one_param_reduction (struct access *repr)
-{
- int total_size, cur_parm_size, agg_size, new_param_count, parm_size_limit;
- bool by_ref;
- tree parm;
-
- parm = repr->base;
- cur_parm_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
- gcc_assert (cur_parm_size > 0);
-
- if (POINTER_TYPE_P (TREE_TYPE (parm)))
- {
- by_ref = true;
- agg_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (parm))));
- }
- else
- {
- by_ref = false;
- agg_size = cur_parm_size;
- }
-
- if (dump_file)
- {
- struct access *acc;
- fprintf (dump_file, "Evaluating PARAM group sizes for ");
- print_generic_expr (dump_file, parm, 0);
- fprintf (dump_file, " (UID: %u): \n", DECL_UID (parm));
- for (acc = repr; acc; acc = acc->next_grp)
- dump_access (dump_file, acc, true);
- }
-
- total_size = 0;
- new_param_count = 0;
-
- for (; repr; repr = repr->next_grp)
- {
- gcc_assert (parm == repr->base);
-
- /* Taking the address of a non-addressable field is verboten. */
- if (by_ref && repr->non_addressable)
- return 0;
-
- /* Do not decompose a non-BLKmode param in a way that would
- create BLKmode params. Especially for by-reference passing
- (thus, pointer-type param) this is hardly worthwhile. */
- if (DECL_MODE (parm) != BLKmode
- && TYPE_MODE (repr->type) == BLKmode)
- return 0;
-
- if (!by_ref || (!repr->grp_maybe_modified
- && !repr->grp_not_necessarilly_dereferenced))
- total_size += repr->size;
- else
- total_size += cur_parm_size;
-
- new_param_count++;
- }
-
- gcc_assert (new_param_count > 0);
-
- if (optimize_function_for_size_p (cfun))
- parm_size_limit = cur_parm_size;
- else
- parm_size_limit = (PARAM_VALUE (PARAM_IPA_SRA_PTR_GROWTH_FACTOR)
- * cur_parm_size);
-
- if (total_size < agg_size
- && total_size <= parm_size_limit)
- {
- if (dump_file)
- fprintf (dump_file, " ....will be split into %i components\n",
- new_param_count);
- return new_param_count;
- }
- else
- return 0;
-}
-
-/* The order of the following enums is important, we need to do extra work for
- UNUSED_PARAMS, BY_VAL_ACCESSES and UNMODIF_BY_REF_ACCESSES. */
-enum ipa_splicing_result { NO_GOOD_ACCESS, UNUSED_PARAMS, BY_VAL_ACCESSES,
- MODIF_BY_REF_ACCESSES, UNMODIF_BY_REF_ACCESSES };
-
-/* Identify representatives of all accesses to all candidate parameters for
- IPA-SRA. Return result based on what representatives have been found. */
-
-static enum ipa_splicing_result
-splice_all_param_accesses (vec<access_p> &representatives)
-{
- enum ipa_splicing_result result = NO_GOOD_ACCESS;
- tree parm;
- struct access *repr;
-
- representatives.create (func_param_count);
-
- for (parm = DECL_ARGUMENTS (current_function_decl);
- parm;
- parm = DECL_CHAIN (parm))
- {
- if (is_unused_scalar_param (parm))
- {
- representatives.quick_push (&no_accesses_representant);
- if (result == NO_GOOD_ACCESS)
- result = UNUSED_PARAMS;
- }
- else if (POINTER_TYPE_P (TREE_TYPE (parm))
- && is_gimple_reg_type (TREE_TYPE (TREE_TYPE (parm)))
- && bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
- {
- repr = unmodified_by_ref_scalar_representative (parm);
- representatives.quick_push (repr);
- if (repr)
- result = UNMODIF_BY_REF_ACCESSES;
- }
- else if (bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
- {
- bool ro_grp = false;
- repr = splice_param_accesses (parm, &ro_grp);
- representatives.quick_push (repr);
-
- if (repr && !no_accesses_p (repr))
- {
- if (POINTER_TYPE_P (TREE_TYPE (parm)))
- {
- if (ro_grp)
- result = UNMODIF_BY_REF_ACCESSES;
- else if (result < MODIF_BY_REF_ACCESSES)
- result = MODIF_BY_REF_ACCESSES;
- }
- else if (result < BY_VAL_ACCESSES)
- result = BY_VAL_ACCESSES;
- }
- else if (no_accesses_p (repr) && (result == NO_GOOD_ACCESS))
- result = UNUSED_PARAMS;
- }
- else
- representatives.quick_push (NULL);
- }
-
- if (result == NO_GOOD_ACCESS)
- {
- representatives.release ();
- return NO_GOOD_ACCESS;
- }
-
- return result;
-}
-
-/* Return the index of BASE in PARMS. Abort if it is not found. */
-
-static inline int
-get_param_index (tree base, vec<tree> parms)
-{
- int i, len;
-
- len = parms.length ();
- for (i = 0; i < len; i++)
- if (parms[i] == base)
- return i;
- gcc_unreachable ();
-}
-
-/* Convert the decisions made at the representative level into compact
- parameter adjustments. REPRESENTATIVES are pointers to first
- representatives of each param accesses, ADJUSTMENTS_COUNT is the expected
- final number of adjustments. */
-
-static ipa_parm_adjustment_vec
-turn_representatives_into_adjustments (vec<access_p> representatives,
- int adjustments_count)
-{
- vec<tree> parms;
- ipa_parm_adjustment_vec adjustments;
- tree parm;
- int i;
-
- gcc_assert (adjustments_count > 0);
- parms = ipa_get_vector_of_formal_parms (current_function_decl);
- adjustments.create (adjustments_count);
- parm = DECL_ARGUMENTS (current_function_decl);
- for (i = 0; i < func_param_count; i++, parm = DECL_CHAIN (parm))
- {
- struct access *repr = representatives[i];
-
- if (!repr || no_accesses_p (repr))
- {
- struct ipa_parm_adjustment adj;
-
- memset (&adj, 0, sizeof (adj));
- adj.base_index = get_param_index (parm, parms);
- adj.base = parm;
- if (!repr)
- adj.op = IPA_PARM_OP_COPY;
- else
- adj.op = IPA_PARM_OP_REMOVE;
- adj.arg_prefix = "ISRA";
- adjustments.quick_push (adj);
- }
- else
- {
- struct ipa_parm_adjustment adj;
- int index = get_param_index (parm, parms);
-
- for (; repr; repr = repr->next_grp)
- {
- memset (&adj, 0, sizeof (adj));
- gcc_assert (repr->base == parm);
- adj.base_index = index;
- adj.base = repr->base;
- adj.type = repr->type;
- adj.alias_ptr_type = reference_alias_ptr_type (repr->expr);
- adj.offset = repr->offset;
- adj.by_ref = (POINTER_TYPE_P (TREE_TYPE (repr->base))
- && (repr->grp_maybe_modified
- || repr->grp_not_necessarilly_dereferenced));
- adj.arg_prefix = "ISRA";
- adjustments.quick_push (adj);
- }
- }
- }
- parms.release ();
- return adjustments;
-}
-
-/* Analyze the collected accesses and produce a plan what to do with the
- parameters in the form of adjustments, NULL meaning nothing. */
-
-static ipa_parm_adjustment_vec
-analyze_all_param_acesses (void)
-{
- enum ipa_splicing_result repr_state;
- bool proceed = false;
- int i, adjustments_count = 0;
- vec<access_p> representatives;
- ipa_parm_adjustment_vec adjustments;
-
- repr_state = splice_all_param_accesses (representatives);
- if (repr_state == NO_GOOD_ACCESS)
- return ipa_parm_adjustment_vec ();
-
- /* If there are any parameters passed by reference which are not modified
- directly, we need to check whether they can be modified indirectly. */
- if (repr_state == UNMODIF_BY_REF_ACCESSES)
- {
- analyze_caller_dereference_legality (representatives);
- analyze_modified_params (representatives);
- }
-
- for (i = 0; i < func_param_count; i++)
- {
- struct access *repr = representatives[i];
-
- if (repr && !no_accesses_p (repr))
- {
- if (repr->grp_scalar_ptr)
- {
- adjustments_count++;
- if (repr->grp_not_necessarilly_dereferenced
- || repr->grp_maybe_modified)
- representatives[i] = NULL;
- else
- {
- proceed = true;
- sra_stats.scalar_by_ref_to_by_val++;
- }
- }
- else
- {
- int new_components = decide_one_param_reduction (repr);
-
- if (new_components == 0)
- {
- representatives[i] = NULL;
- adjustments_count++;
- }
- else
- {
- adjustments_count += new_components;
- sra_stats.aggregate_params_reduced++;
- sra_stats.param_reductions_created += new_components;
- proceed = true;
- }
- }
- }
- else
- {
- if (no_accesses_p (repr))
- {
- proceed = true;
- sra_stats.deleted_unused_parameters++;
- }
- adjustments_count++;
- }
- }
-
- if (!proceed && dump_file)
- fprintf (dump_file, "NOT proceeding to change params.\n");
-
- if (proceed)
- adjustments = turn_representatives_into_adjustments (representatives,
- adjustments_count);
- else
- adjustments = ipa_parm_adjustment_vec ();
-
- representatives.release ();
- return adjustments;
-}
-
-/* If a parameter replacement identified by ADJ does not yet exist in the form
- of declaration, create it and record it, otherwise return the previously
- created one. */
-
-static tree
-get_replaced_param_substitute (struct ipa_parm_adjustment *adj)
-{
- tree repl;
- if (!adj->new_ssa_base)
- {
- char *pretty_name = make_fancy_name (adj->base);
-
- repl = create_tmp_reg (TREE_TYPE (adj->base), "ISR");
- DECL_NAME (repl) = get_identifier (pretty_name);
- obstack_free (&name_obstack, pretty_name);
-
- adj->new_ssa_base = repl;
- }
- else
- repl = adj->new_ssa_base;
- return repl;
-}
-
-/* Find the first adjustment for a particular parameter BASE in a vector of
- ADJUSTMENTS which is not a copy_param. Return NULL if there is no such
- adjustment. */
-
-static struct ipa_parm_adjustment *
-get_adjustment_for_base (ipa_parm_adjustment_vec adjustments, tree base)
-{
- int i, len;
-
- len = adjustments.length ();
- for (i = 0; i < len; i++)
- {
- struct ipa_parm_adjustment *adj;
-
- adj = &adjustments[i];
- if (adj->op != IPA_PARM_OP_COPY && adj->base == base)
- return adj;
- }
-
- return NULL;
-}
-
-/* If the statement STMT defines an SSA_NAME of a parameter which is to be
- removed because its value is not used, replace the SSA_NAME with a one
- relating to a created VAR_DECL together all of its uses and return true.
- ADJUSTMENTS is a pointer to an adjustments vector. */
-
-static bool
-replace_removed_params_ssa_names (gimple stmt,
- ipa_parm_adjustment_vec adjustments)
-{
- struct ipa_parm_adjustment *adj;
- tree lhs, decl, repl, name;
-
- if (gimple_code (stmt) == GIMPLE_PHI)
- lhs = gimple_phi_result (stmt);
- else if (is_gimple_assign (stmt))
- lhs = gimple_assign_lhs (stmt);
- else if (is_gimple_call (stmt))
- lhs = gimple_call_lhs (stmt);
- else
- gcc_unreachable ();
-
- if (TREE_CODE (lhs) != SSA_NAME)
- return false;
-
- decl = SSA_NAME_VAR (lhs);
- if (decl == NULL_TREE
- || TREE_CODE (decl) != PARM_DECL)
- return false;
-
- adj = get_adjustment_for_base (adjustments, decl);
- if (!adj)
- return false;
-
- repl = get_replaced_param_substitute (adj);
- name = make_ssa_name (repl, stmt);
-
- if (dump_file)
- {
- fprintf (dump_file, "replacing an SSA name of a removed param ");
- print_generic_expr (dump_file, lhs, 0);
- fprintf (dump_file, " with ");
- print_generic_expr (dump_file, name, 0);
- fprintf (dump_file, "\n");
- }
-
- if (is_gimple_assign (stmt))
- gimple_assign_set_lhs (stmt, name);
- else if (is_gimple_call (stmt))
- gimple_call_set_lhs (stmt, name);
- else
- gimple_phi_set_result (as_a <gphi *> (stmt), name);
-
- replace_uses_by (lhs, name);
- release_ssa_name (lhs);
- return true;
-}
-
-/* If the statement STMT contains any expressions that need to replaced with a
- different one as noted by ADJUSTMENTS, do so. Handle any potential type
- incompatibilities (GSI is used to accommodate conversion statements and must
- point to the statement). Return true iff the statement was modified. */
-
-static bool
-sra_ipa_modify_assign (gimple stmt, gimple_stmt_iterator *gsi,
- ipa_parm_adjustment_vec adjustments)
-{
- tree *lhs_p, *rhs_p;
- bool any;
-
- if (!gimple_assign_single_p (stmt))
- return false;
-
- rhs_p = gimple_assign_rhs1_ptr (stmt);
- lhs_p = gimple_assign_lhs_ptr (stmt);
-
- any = ipa_modify_expr (rhs_p, false, adjustments);
- any |= ipa_modify_expr (lhs_p, false, adjustments);
- if (any)
- {
- tree new_rhs = NULL_TREE;
-
- if (!useless_type_conversion_p (TREE_TYPE (*lhs_p), TREE_TYPE (*rhs_p)))
- {
- if (TREE_CODE (*rhs_p) == CONSTRUCTOR)
- {
- /* V_C_Es of constructors can cause trouble (PR 42714). */
- if (is_gimple_reg_type (TREE_TYPE (*lhs_p)))
- *rhs_p = build_zero_cst (TREE_TYPE (*lhs_p));
- else
- *rhs_p = build_constructor (TREE_TYPE (*lhs_p),
- NULL);
- }
- else
- new_rhs = fold_build1_loc (gimple_location (stmt),
- VIEW_CONVERT_EXPR, TREE_TYPE (*lhs_p),
- *rhs_p);
- }
- else if (REFERENCE_CLASS_P (*rhs_p)
- && is_gimple_reg_type (TREE_TYPE (*lhs_p))
- && !is_gimple_reg (*lhs_p))
- /* This can happen when an assignment in between two single field
- structures is turned into an assignment in between two pointers to
- scalars (PR 42237). */
- new_rhs = *rhs_p;
-
- if (new_rhs)
- {
- tree tmp = force_gimple_operand_gsi (gsi, new_rhs, true, NULL_TREE,
- true, GSI_SAME_STMT);
-
- gimple_assign_set_rhs_from_tree (gsi, tmp);
- }
-
- return true;
- }
-
- return false;
-}
-
-/* Traverse the function body and all modifications as described in
- ADJUSTMENTS. Return true iff the CFG has been changed. */
-
-bool
-ipa_sra_modify_function_body (ipa_parm_adjustment_vec adjustments)
-{
- bool cfg_changed = false;
- basic_block bb;
-
- FOR_EACH_BB_FN (bb, cfun)
- {
- gimple_stmt_iterator gsi;
-
- for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
- replace_removed_params_ssa_names (gsi_stmt (gsi), adjustments);
-
- gsi = gsi_start_bb (bb);
- while (!gsi_end_p (gsi))
- {
- gimple stmt = gsi_stmt (gsi);
- bool modified = false;
- tree *t;
- unsigned i;
-
- switch (gimple_code (stmt))
- {
- case GIMPLE_RETURN:
- t = gimple_return_retval_ptr (as_a <greturn *> (stmt));
- if (*t != NULL_TREE)
- modified |= ipa_modify_expr (t, true, adjustments);
- break;
-
- case GIMPLE_ASSIGN:
- modified |= sra_ipa_modify_assign (stmt, &gsi, adjustments);
- modified |= replace_removed_params_ssa_names (stmt, adjustments);
- break;
-
- case GIMPLE_CALL:
- /* Operands must be processed before the lhs. */
- for (i = 0; i < gimple_call_num_args (stmt); i++)
- {
- t = gimple_call_arg_ptr (stmt, i);
- modified |= ipa_modify_expr (t, true, adjustments);
- }
-
- if (gimple_call_lhs (stmt))
- {
- t = gimple_call_lhs_ptr (stmt);
- modified |= ipa_modify_expr (t, false, adjustments);
- modified |= replace_removed_params_ssa_names (stmt,
- adjustments);
- }
- break;
-
- case GIMPLE_ASM:
- {
- gasm *asm_stmt = as_a <gasm *> (stmt);
- for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
- {
- t = &TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
- modified |= ipa_modify_expr (t, true, adjustments);
- }
- for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
- {
- t = &TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
- modified |= ipa_modify_expr (t, false, adjustments);
- }
- }
- break;
-
- default:
- break;
- }
-
- if (modified)
- {
- update_stmt (stmt);
- if (maybe_clean_eh_stmt (stmt)
- && gimple_purge_dead_eh_edges (gimple_bb (stmt)))
- cfg_changed = true;
- }
- gsi_next (&gsi);
- }
- }
-
- return cfg_changed;
-}
-
-/* Call gimple_debug_bind_reset_value on all debug statements describing
- gimple register parameters that are being removed or replaced. */
-
-static void
-sra_ipa_reset_debug_stmts (ipa_parm_adjustment_vec adjustments)
-{
- int i, len;
- gimple_stmt_iterator *gsip = NULL, gsi;
-
- if (MAY_HAVE_DEBUG_STMTS && single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
- {
- gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
- gsip = &gsi;
- }
- len = adjustments.length ();
- for (i = 0; i < len; i++)
- {
- struct ipa_parm_adjustment *adj;
- imm_use_iterator ui;
- gimple stmt;
- gdebug *def_temp;
- tree name, vexpr, copy = NULL_TREE;
- use_operand_p use_p;
-
- adj = &adjustments[i];
- if (adj->op == IPA_PARM_OP_COPY || !is_gimple_reg (adj->base))
- continue;
- name = ssa_default_def (cfun, adj->base);
- vexpr = NULL;
- if (name)
- FOR_EACH_IMM_USE_STMT (stmt, ui, name)
- {
- if (gimple_clobber_p (stmt))
- {
- gimple_stmt_iterator cgsi = gsi_for_stmt (stmt);
- unlink_stmt_vdef (stmt);
- gsi_remove (&cgsi, true);
- release_defs (stmt);
- continue;
- }
- /* All other users must have been removed by
- ipa_sra_modify_function_body. */
- gcc_assert (is_gimple_debug (stmt));
- if (vexpr == NULL && gsip != NULL)
- {
- gcc_assert (TREE_CODE (adj->base) == PARM_DECL);
- vexpr = make_node (DEBUG_EXPR_DECL);
- def_temp = gimple_build_debug_source_bind (vexpr, adj->base,
- NULL);
- DECL_ARTIFICIAL (vexpr) = 1;
- TREE_TYPE (vexpr) = TREE_TYPE (name);
- DECL_MODE (vexpr) = DECL_MODE (adj->base);
- gsi_insert_before (gsip, def_temp, GSI_SAME_STMT);
- }
- if (vexpr)
- {
- FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
- SET_USE (use_p, vexpr);
- }
- else
- gimple_debug_bind_reset_value (stmt);
- update_stmt (stmt);
- }
- /* Create a VAR_DECL for debug info purposes. */
- if (!DECL_IGNORED_P (adj->base))
- {
- copy = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
- VAR_DECL, DECL_NAME (adj->base),
- TREE_TYPE (adj->base));
- if (DECL_PT_UID_SET_P (adj->base))
- SET_DECL_PT_UID (copy, DECL_PT_UID (adj->base));
- TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (adj->base);
- TREE_READONLY (copy) = TREE_READONLY (adj->base);
- TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (adj->base);
- DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (adj->base);
- DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (adj->base);
- DECL_IGNORED_P (copy) = DECL_IGNORED_P (adj->base);
- DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (adj->base);
- DECL_SEEN_IN_BIND_EXPR_P (copy) = 1;
- SET_DECL_RTL (copy, 0);
- TREE_USED (copy) = 1;
- DECL_CONTEXT (copy) = current_function_decl;
- add_local_decl (cfun, copy);
- DECL_CHAIN (copy) =
- BLOCK_VARS (DECL_INITIAL (current_function_decl));
- BLOCK_VARS (DECL_INITIAL (current_function_decl)) = copy;
- }
- if (gsip != NULL && copy && target_for_debug_bind (adj->base))
- {
- gcc_assert (TREE_CODE (adj->base) == PARM_DECL);
- if (vexpr)
- def_temp = gimple_build_debug_bind (copy, vexpr, NULL);
- else
- def_temp = gimple_build_debug_source_bind (copy, adj->base,
- NULL);
- gsi_insert_before (gsip, def_temp, GSI_SAME_STMT);
- }
- }
-}
-
-/* Return false if all callers have at least as many actual arguments as there
- are formal parameters in the current function and that their types
- match. */
-
-static bool
-some_callers_have_mismatched_arguments_p (struct cgraph_node *node,
- void *data ATTRIBUTE_UNUSED)
-{
- struct cgraph_edge *cs;
- for (cs = node->callers; cs; cs = cs->next_caller)
- if (!cs->call_stmt || !callsite_arguments_match_p (cs->call_stmt))
- return true;
-
- return false;
-}
-
-/* Return false if all callers have vuse attached to a call statement. */
-
-static bool
-some_callers_have_no_vuse_p (struct cgraph_node *node,
- void *data ATTRIBUTE_UNUSED)
-{
- struct cgraph_edge *cs;
- for (cs = node->callers; cs; cs = cs->next_caller)
- if (!cs->call_stmt || !gimple_vuse (cs->call_stmt))
- return true;
-
- return false;
-}
-
-/* Convert all callers of NODE. */
-
-static bool
-convert_callers_for_node (struct cgraph_node *node,
- void *data)
-{
- ipa_parm_adjustment_vec *adjustments = (ipa_parm_adjustment_vec *) data;
- bitmap recomputed_callers = BITMAP_ALLOC (NULL);
- struct cgraph_edge *cs;
-
- for (cs = node->callers; cs; cs = cs->next_caller)
- {
- push_cfun (DECL_STRUCT_FUNCTION (cs->caller->decl));
-
- if (dump_file)
- fprintf (dump_file, "Adjusting call %s/%i -> %s/%i\n",
- xstrdup (cs->caller->name ()),
- cs->caller->order,
- xstrdup (cs->callee->name ()),
- cs->callee->order);
-
- ipa_modify_call_arguments (cs, cs->call_stmt, *adjustments);
-
- pop_cfun ();
- }
-
- for (cs = node->callers; cs; cs = cs->next_caller)
- if (bitmap_set_bit (recomputed_callers, cs->caller->uid)
- && gimple_in_ssa_p (DECL_STRUCT_FUNCTION (cs->caller->decl)))
- compute_inline_parameters (cs->caller, true);
- BITMAP_FREE (recomputed_callers);
-
- return true;
-}
-
-/* Convert all callers of NODE to pass parameters as given in ADJUSTMENTS. */
-
-static void
-convert_callers (struct cgraph_node *node, tree old_decl,
- ipa_parm_adjustment_vec adjustments)
-{
- basic_block this_block;
-
- node->call_for_symbol_and_aliases (convert_callers_for_node,
- &adjustments, false);
-
- if (!encountered_recursive_call)
- return;
-
- FOR_EACH_BB_FN (this_block, cfun)
- {
- gimple_stmt_iterator gsi;
-
- for (gsi = gsi_start_bb (this_block); !gsi_end_p (gsi); gsi_next (&gsi))
- {
- gcall *stmt;
- tree call_fndecl;
- stmt = dyn_cast <gcall *> (gsi_stmt (gsi));
- if (!stmt)
- continue;
- call_fndecl = gimple_call_fndecl (stmt);
- if (call_fndecl == old_decl)
- {
- if (dump_file)
- fprintf (dump_file, "Adjusting recursive call");
- gimple_call_set_fndecl (stmt, node->decl);
- ipa_modify_call_arguments (NULL, stmt, adjustments);
- }
- }
- }
-
- return;
-}
-
-/* Perform all the modification required in IPA-SRA for NODE to have parameters
- as given in ADJUSTMENTS. Return true iff the CFG has been changed. */
-
-static bool
-modify_function (struct cgraph_node *node, ipa_parm_adjustment_vec adjustments)
-{
- struct cgraph_node *new_node;
- bool cfg_changed;
-
- cgraph_edge::rebuild_edges ();
- free_dominance_info (CDI_DOMINATORS);
- pop_cfun ();
-
- /* This must be done after rebuilding cgraph edges for node above.
- Otherwise any recursive calls to node that are recorded in
- redirect_callers will be corrupted. */
- vec<cgraph_edge *> redirect_callers = node->collect_callers ();
- new_node = node->create_version_clone_with_body (redirect_callers, NULL,
- NULL, false, NULL, NULL,
- "isra");
- redirect_callers.release ();
-
- push_cfun (DECL_STRUCT_FUNCTION (new_node->decl));
- ipa_modify_formal_parameters (current_function_decl, adjustments);
- cfg_changed = ipa_sra_modify_function_body (adjustments);
- sra_ipa_reset_debug_stmts (adjustments);
- convert_callers (new_node, node->decl, adjustments);
- new_node->make_local ();
- return cfg_changed;
-}
-
-/* Means of communication between ipa_sra_check_caller and
- ipa_sra_preliminary_function_checks. */
-
-struct ipa_sra_check_caller_data
-{
- bool has_callers;
- bool bad_arg_alignment;
- bool has_thunk;
-};
-
-/* If NODE has a caller, mark that fact in DATA which is pointer to
- ipa_sra_check_caller_data. Also check all aggregate arguments in all known
- calls if they are unit aligned and if not, set the appropriate flag in DATA
- too. */
-
-static bool
-ipa_sra_check_caller (struct cgraph_node *node, void *data)
-{
- if (!node->callers)
- return false;
-
- struct ipa_sra_check_caller_data *iscc;
- iscc = (struct ipa_sra_check_caller_data *) data;
- iscc->has_callers = true;
-
- for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
- {
- if (cs->caller->thunk.thunk_p)
- {
- iscc->has_thunk = true;
- return true;
- }
- gimple call_stmt = cs->call_stmt;
- unsigned count = gimple_call_num_args (call_stmt);
- for (unsigned i = 0; i < count; i++)
- {
- tree arg = gimple_call_arg (call_stmt, i);
- if (is_gimple_reg (arg))
- continue;
-
- tree offset;
- HOST_WIDE_INT bitsize, bitpos;
- machine_mode mode;
- int unsignedp, volatilep = 0;
- get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
- &unsignedp, &volatilep, false);
- if (bitpos % BITS_PER_UNIT)
- {
- iscc->bad_arg_alignment = true;
- return true;
- }
- }
- }
-
- return false;
-}
-
-/* Return false the function is apparently unsuitable for IPA-SRA based on it's
- attributes, return true otherwise. NODE is the cgraph node of the current
- function. */
-
-static bool
-ipa_sra_preliminary_function_checks (struct cgraph_node *node)
-{
- if (!node->can_be_local_p ())
- {
- if (dump_file)
- fprintf (dump_file, "Function not local to this compilation unit.\n");
- return false;
- }
-
- if (!node->local.can_change_signature)
- {
- if (dump_file)
- fprintf (dump_file, "Function can not change signature.\n");
- return false;
- }
-
- if (!tree_versionable_function_p (node->decl))
- {
- if (dump_file)
- fprintf (dump_file, "Function is not versionable.\n");
- return false;
- }
-
- if (!opt_for_fn (node->decl, optimize)
- || !opt_for_fn (node->decl, flag_ipa_sra))
- {
- if (dump_file)
- fprintf (dump_file, "Function not optimized.\n");
- return false;
- }
-
- if (DECL_VIRTUAL_P (current_function_decl))
- {
- if (dump_file)
- fprintf (dump_file, "Function is a virtual method.\n");
- return false;
- }
-
- if ((DECL_ONE_ONLY (node->decl) || DECL_EXTERNAL (node->decl))
- && inline_summaries->get (node)->size >= MAX_INLINE_INSNS_AUTO)
- {
- if (dump_file)
- fprintf (dump_file, "Function too big to be made truly local.\n");
- return false;
- }
-
- if (cfun->stdarg)
- {
- if (dump_file)
- fprintf (dump_file, "Function uses stdarg. \n");
- return false;
- }
-
- if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl)))
- return false;
-
- if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
- {
- if (dump_file)
- fprintf (dump_file, "Always inline function will be inlined "
- "anyway. \n");
- return false;
- }
-
- struct ipa_sra_check_caller_data iscc;
- memset (&iscc, 0, sizeof(iscc));
- node->call_for_symbol_and_aliases (ipa_sra_check_caller, &iscc, true);
- if (!iscc.has_callers)
- {
- if (dump_file)
- fprintf (dump_file,
- "Function has no callers in this compilation unit.\n");
- return false;
- }
-
- if (iscc.bad_arg_alignment)
- {
- if (dump_file)
- fprintf (dump_file,
- "A function call has an argument with non-unit alignment.\n");
- return false;
- }
-
- if (iscc.has_thunk)
- {
- if (dump_file)
- fprintf (dump_file,
- "A has thunk.\n");
- return false;
- }
-
- return true;
-}
-
-/* Perform early interprocedural SRA. */
-
-static unsigned int
-ipa_early_sra (void)
-{
- struct cgraph_node *node = cgraph_node::get (current_function_decl);
- ipa_parm_adjustment_vec adjustments;
- int ret = 0;
-
- if (!ipa_sra_preliminary_function_checks (node))
- return 0;
-
- sra_initialize ();
- sra_mode = SRA_MODE_EARLY_IPA;
-
- if (!find_param_candidates ())
- {
- if (dump_file)
- fprintf (dump_file, "Function has no IPA-SRA candidates.\n");
- goto simple_out;
- }
-
- if (node->call_for_symbol_and_aliases
- (some_callers_have_mismatched_arguments_p, NULL, true))
- {
- if (dump_file)
- fprintf (dump_file, "There are callers with insufficient number of "
- "arguments or arguments with type mismatches.\n");
- goto simple_out;
- }
-
- if (node->call_for_symbol_and_aliases
- (some_callers_have_no_vuse_p, NULL, true))
- {
- if (dump_file)
- fprintf (dump_file, "There are callers with no VUSE attached "
- "to a call stmt.\n");
- goto simple_out;
- }
-
- bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
- func_param_count
- * last_basic_block_for_fn (cfun));
- final_bbs = BITMAP_ALLOC (NULL);
-
- scan_function ();
- if (encountered_apply_args)
- {
- if (dump_file)
- fprintf (dump_file, "Function calls __builtin_apply_args().\n");
- goto out;
- }
-
- if (encountered_unchangable_recursive_call)
- {
- if (dump_file)
- fprintf (dump_file, "Function calls itself with insufficient "
- "number of arguments.\n");
- goto out;
- }
-
- adjustments = analyze_all_param_acesses ();
- if (!adjustments.exists ())
- goto out;
- if (dump_file)
- ipa_dump_param_adjustments (dump_file, adjustments, current_function_decl);
-
- if (modify_function (node, adjustments))
- ret = TODO_update_ssa | TODO_cleanup_cfg;
- else
- ret = TODO_update_ssa;
- adjustments.release ();
-
- statistics_counter_event (cfun, "Unused parameters deleted",
- sra_stats.deleted_unused_parameters);
- statistics_counter_event (cfun, "Scalar parameters converted to by-value",
- sra_stats.scalar_by_ref_to_by_val);
- statistics_counter_event (cfun, "Aggregate parameters broken up",
- sra_stats.aggregate_params_reduced);
- statistics_counter_event (cfun, "Aggregate parameter components created",
- sra_stats.param_reductions_created);
-
- out:
- BITMAP_FREE (final_bbs);
- free (bb_dereferences);
- simple_out:
- sra_deinitialize ();
- return ret;
-}
-
-namespace {
-
-const pass_data pass_data_early_ipa_sra =
-{
- GIMPLE_PASS, /* type */
- "eipa_sra", /* name */
- OPTGROUP_NONE, /* optinfo_flags */
- TV_IPA_SRA, /* tv_id */
- 0, /* properties_required */
- 0, /* properties_provided */
- 0, /* properties_destroyed */
- 0, /* todo_flags_start */
- TODO_dump_symtab, /* todo_flags_finish */
-};
-
-class pass_early_ipa_sra : public gimple_opt_pass
-{
-public:
- pass_early_ipa_sra (gcc::context *ctxt)
- : gimple_opt_pass (pass_data_early_ipa_sra, ctxt)
- {}
-
- /* opt_pass methods: */
- virtual bool gate (function *) { return flag_ipa_sra && dbg_cnt (eipa_sra); }
- virtual unsigned int execute (function *) { return ipa_early_sra (); }
-
-}; // class pass_early_ipa_sra
-
-} // anon namespace
-
-gimple_opt_pass *
-make_pass_early_ipa_sra (gcc::context *ctxt)
-{
- return new pass_early_ipa_sra (ctxt);
-}
projects / thirdparty / gcc.git / blobdiff