/* Scalar Replacement of Aggregates (SRA) converts some structure
references into scalar references, exposing them to the scalar
optimizers.
- Copyright (C) 2008-2018 Free Software Foundation, Inc.
+ Copyright (C) 2008-2020 Free Software Foundation, Inc.
Contributed by Martin Jambor <mjambor@suse.cz>
This file is part of GCC.
#include "tree-cfg.h"
#include "tree-dfa.h"
#include "tree-ssa.h"
-#include "symbol-summary.h"
-#include "ipa-param-manipulation.h"
-#include "ipa-prop.h"
-#include "params.h"
#include "dbgcnt.h"
-#include "tree-inline.h"
-#include "ipa-fnsummary.h"
-#include "ipa-utils.h"
#include "builtins.h"
+#include "tree-sra.h"
+
/* Enumeration of all aggregate reductions we can do. */
enum sra_mode { SRA_MODE_EARLY_IPA, /* early call regularization */
struct access *first_child;
/* In intraprocedural SRA, pointer to the next sibling in the access tree as
- described above. In IPA-SRA this is a pointer to the next access
- belonging to the same group (having the same representative). */
+ described above. */
struct access *next_sibling;
/* Pointers to the first and last element in the linked list of assign
- links. */
- struct assign_link *first_link, *last_link;
+ links for propagation from LHS to RHS. */
+ struct assign_link *first_rhs_link, *last_rhs_link;
+
+ /* Pointers to the first and last element in the linked list of assign
+ links for propagation from LHS to RHS. */
+ struct assign_link *first_lhs_link, *last_lhs_link;
- /* Pointer to the next access in the work queue. */
- struct access *next_queued;
+ /* Pointer to the next access in the work queues. */
+ struct access *next_rhs_queued, *next_lhs_queued;
/* Replacement variable for this access "region." Never to be accessed
directly, always only by the means of get_access_replacement() and only
when grp_to_be_replaced flag is set. */
tree replacement_decl;
- /* Is this access an access to a non-addressable field? */
- unsigned non_addressable : 1;
-
/* Is this access made in reverse storage order? */
unsigned reverse : 1;
/* Is this particular access write access? */
unsigned write : 1;
- /* Is this access currently in the work queue? */
- unsigned grp_queued : 1;
+ /* Is this access currently in the rhs work queue? */
+ unsigned grp_rhs_queued : 1;
+
+ /* Is this access currently in the lhs work queue? */
+ unsigned grp_lhs_queued : 1;
/* Does this group contain a write access? This flag is propagated down the
access tree. */
is not propagated in the access tree in any direction. */
unsigned grp_scalar_write : 1;
- /* Is this access an artificial one created to scalarize some record
- entirely? */
+ /* In a root of an access tree, true means that the entire tree should be
+ totally scalarized - that all scalar leafs should be scalarized and
+ non-root grp_total_scalarization accesses should be honored. Otherwise,
+ non-root accesses with grp_total_scalarization should never get scalar
+ replacements. */
unsigned grp_total_scalarization : 1;
/* Other passes of the analysis use this bit to make function
access tree. */
unsigned grp_unscalarized_data : 1;
+ /* Set if all accesses in the group consist of the same chain of
+ COMPONENT_REFs and ARRAY_REFs. */
+ unsigned grp_same_access_path : 1;
+
/* Does this access and/or group contain a write access through a
BIT_FIELD_REF? */
unsigned grp_partial_lhs : 1;
/* Should TREE_NO_WARNING of a replacement be set? */
unsigned grp_no_warning : 1;
-
- /* Is it possible that the group refers to data which might be (directly or
- otherwise) modified? */
- unsigned grp_maybe_modified : 1;
-
- /* Set when this is a representative of a pointer to scalar (i.e. by
- reference) parameter which we consider for turning into a plain scalar
- (i.e. a by value parameter). */
- unsigned grp_scalar_ptr : 1;
-
- /* Set when we discover that this pointer is not safe to dereference in the
- caller. */
- unsigned grp_not_necessarilly_dereferenced : 1;
};
typedef struct access *access_p;
static object_allocator<struct access> access_pool ("SRA accesses");
/* A structure linking lhs and rhs accesses from an aggregate assignment. They
- are used to propagate subaccesses from rhs to lhs as long as they don't
- conflict with what is already there. */
+ are used to propagate subaccesses from rhs to lhs and vice versa as long as
+ they don't conflict with what is already there. In the RHS->LHS direction,
+ we also propagate grp_write flag to lazily mark that the access contains any
+ meaningful data. */
struct assign_link
{
struct access *lacc, *racc;
- struct assign_link *next;
+ struct assign_link *next_rhs, *next_lhs;
};
/* Alloc pool for allocating assign link structures. */
/* Base (tree) -> Vector (vec<access_p> *) map. */
static hash_map<tree, auto_vec<access_p> > *base_access_vec;
+/* Hash to limit creation of artificial accesses */
+static hash_map<tree, unsigned> *propagation_budget;
+
/* Candidate hash table helpers. */
struct uid_decl_hasher : nofree_ptr_hash <tree_node>
/* Head of a linked list of accesses that need to have its subaccesses
propagated to their assignment counterparts. */
-static struct access *work_queue_head;
-
-/* Number of parameters of the analyzed function when doing early ipa SRA. */
-static int func_param_count;
-
-/* scan_function sets the following to true if it encounters a call to
- __builtin_apply_args. */
-static bool encountered_apply_args;
-
-/* Set by scan_function when it finds a recursive call. */
-static bool encountered_recursive_call;
-
-/* Set by scan_function when it finds a recursive call with less actual
- arguments than formal parameters.. */
-static bool encountered_unchangable_recursive_call;
-
-/* This is a table in which for each basic block and parameter there is a
- distance (offset + size) in that parameter which is dereferenced and
- accessed in that BB. */
-static HOST_WIDE_INT *bb_dereferences;
-/* Bitmap of BBs that can cause the function to "stop" progressing by
- returning, throwing externally, looping infinitely or calling a function
- which might abort etc.. */
-static bitmap final_bbs;
-
-/* Representative of no accesses at all. */
-static struct access no_accesses_representant;
-
-/* Predicate to test the special value. */
-
-static inline bool
-no_accesses_p (struct access *access)
-{
- return access == &no_accesses_representant;
-}
+static struct access *rhs_work_queue_head, *lhs_work_queue_head;
/* Dump contents of ACCESS to file F in a human friendly way. If GRP is true,
representative fields are dumped, otherwise those which only describe the
print_generic_expr (f, access->expr);
fprintf (f, ", type = ");
print_generic_expr (f, access->type);
- fprintf (f, ", non_addressable = %d, reverse = %d",
- access->non_addressable, access->reverse);
+ fprintf (f, ", reverse = %d", access->reverse);
if (grp)
fprintf (f, ", grp_read = %d, grp_write = %d, grp_assignment_read = %d, "
"grp_assignment_write = %d, grp_scalar_read = %d, "
"grp_scalar_write = %d, grp_total_scalarization = %d, "
"grp_hint = %d, grp_covered = %d, "
"grp_unscalarizable_region = %d, grp_unscalarized_data = %d, "
- "grp_partial_lhs = %d, grp_to_be_replaced = %d, "
- "grp_to_be_debug_replaced = %d, grp_maybe_modified = %d, "
- "grp_not_necessarilly_dereferenced = %d\n",
+ "grp_same_access_path = %d, grp_partial_lhs = %d, "
+ "grp_to_be_replaced = %d, grp_to_be_debug_replaced = %d}\n",
access->grp_read, access->grp_write, access->grp_assignment_read,
access->grp_assignment_write, access->grp_scalar_read,
access->grp_scalar_write, access->grp_total_scalarization,
access->grp_hint, access->grp_covered,
access->grp_unscalarizable_region, access->grp_unscalarized_data,
- access->grp_partial_lhs, access->grp_to_be_replaced,
- access->grp_to_be_debug_replaced, access->grp_maybe_modified,
- access->grp_not_necessarilly_dereferenced);
+ access->grp_same_access_path, access->grp_partial_lhs,
+ access->grp_to_be_replaced, access->grp_to_be_debug_replaced);
else
fprintf (f, ", write = %d, grp_total_scalarization = %d, "
- "grp_partial_lhs = %d\n",
+ "grp_partial_lhs = %d}\n",
access->write, access->grp_total_scalarization,
access->grp_partial_lhs);
}
access = child;
}
+ /* Total scalarization does not replace single field structures with their
+ single field but rather creates an access for them underneath. Look for
+ it. */
+ if (access)
+ while (access->first_child
+ && access->first_child->offset == offset
+ && access->first_child->size == size)
+ access = access->first_child;
+
return access;
}
}
/* Add LINK to the linked list of assign links of RACC. */
+
static void
add_link_to_rhs (struct access *racc, struct assign_link *link)
{
gcc_assert (link->racc == racc);
- if (!racc->first_link)
+ if (!racc->first_rhs_link)
{
- gcc_assert (!racc->last_link);
- racc->first_link = link;
+ gcc_assert (!racc->last_rhs_link);
+ racc->first_rhs_link = link;
}
else
- racc->last_link->next = link;
+ racc->last_rhs_link->next_rhs = link;
- racc->last_link = link;
- link->next = NULL;
+ racc->last_rhs_link = link;
+ link->next_rhs = NULL;
}
-/* Move all link structures in their linked list in OLD_RACC to the linked list
- in NEW_RACC. */
+/* Add LINK to the linked list of lhs assign links of LACC. */
+
static void
-relink_to_new_repr (struct access *new_racc, struct access *old_racc)
+add_link_to_lhs (struct access *lacc, struct assign_link *link)
{
- if (!old_racc->first_link)
+ gcc_assert (link->lacc == lacc);
+
+ if (!lacc->first_lhs_link)
{
- gcc_assert (!old_racc->last_link);
- return;
+ gcc_assert (!lacc->last_lhs_link);
+ lacc->first_lhs_link = link;
}
+ else
+ lacc->last_lhs_link->next_lhs = link;
- if (new_racc->first_link)
+ lacc->last_lhs_link = link;
+ link->next_lhs = NULL;
+}
+
+/* Move all link structures in their linked list in OLD_ACC to the linked list
+ in NEW_ACC. */
+static void
+relink_to_new_repr (struct access *new_acc, struct access *old_acc)
+{
+ if (old_acc->first_rhs_link)
{
- gcc_assert (!new_racc->last_link->next);
- gcc_assert (!old_racc->last_link || !old_racc->last_link->next);
- new_racc->last_link->next = old_racc->first_link;
- new_racc->last_link = old_racc->last_link;
+ if (new_acc->first_rhs_link)
+ {
+ gcc_assert (!new_acc->last_rhs_link->next_rhs);
+ gcc_assert (!old_acc->last_rhs_link
+ || !old_acc->last_rhs_link->next_rhs);
+
+ new_acc->last_rhs_link->next_rhs = old_acc->first_rhs_link;
+ new_acc->last_rhs_link = old_acc->last_rhs_link;
+ }
+ else
+ {
+ gcc_assert (!new_acc->last_rhs_link);
+
+ new_acc->first_rhs_link = old_acc->first_rhs_link;
+ new_acc->last_rhs_link = old_acc->last_rhs_link;
+ }
+ old_acc->first_rhs_link = old_acc->last_rhs_link = NULL;
}
else
+ gcc_assert (!old_acc->last_rhs_link);
+
+ if (old_acc->first_lhs_link)
{
- gcc_assert (!new_racc->last_link);
- new_racc->first_link = old_racc->first_link;
- new_racc->last_link = old_racc->last_link;
+ if (new_acc->first_lhs_link)
+ {
+ gcc_assert (!new_acc->last_lhs_link->next_lhs);
+ gcc_assert (!old_acc->last_lhs_link
+ || !old_acc->last_lhs_link->next_lhs);
+
+ new_acc->last_lhs_link->next_lhs = old_acc->first_lhs_link;
+ new_acc->last_lhs_link = old_acc->last_lhs_link;
+ }
+ else
+ {
+ gcc_assert (!new_acc->last_lhs_link);
+
+ new_acc->first_lhs_link = old_acc->first_lhs_link;
+ new_acc->last_lhs_link = old_acc->last_lhs_link;
+ }
+ old_acc->first_lhs_link = old_acc->last_lhs_link = NULL;
+ }
+ else
+ gcc_assert (!old_acc->last_lhs_link);
+
+}
+
+/* Add ACCESS to the work to queue for propagation of subaccesses from RHS to
+ LHS (which is actually a stack). */
+
+static void
+add_access_to_rhs_work_queue (struct access *access)
+{
+ if (access->first_rhs_link && !access->grp_rhs_queued)
+ {
+ gcc_assert (!access->next_rhs_queued);
+ access->next_rhs_queued = rhs_work_queue_head;
+ access->grp_rhs_queued = 1;
+ rhs_work_queue_head = access;
}
- old_racc->first_link = old_racc->last_link = NULL;
}
-/* Add ACCESS to the work queue (which is actually a stack). */
+/* Add ACCESS to the work to queue for propagation of subaccesses from LHS to
+ RHS (which is actually a stack). */
static void
-add_access_to_work_queue (struct access *access)
+add_access_to_lhs_work_queue (struct access *access)
{
- if (access->first_link && !access->grp_queued)
+ if (access->first_lhs_link && !access->grp_lhs_queued)
{
- gcc_assert (!access->next_queued);
- access->next_queued = work_queue_head;
- access->grp_queued = 1;
- work_queue_head = access;
+ gcc_assert (!access->next_lhs_queued);
+ access->next_lhs_queued = lhs_work_queue_head;
+ access->grp_lhs_queued = 1;
+ lhs_work_queue_head = access;
}
}
-/* Pop an access from the work queue, and return it, assuming there is one. */
+/* Pop an access from the work queue for propagating from RHS to LHS, and
+ return it, assuming there is one. */
static struct access *
-pop_access_from_work_queue (void)
+pop_access_from_rhs_work_queue (void)
{
- struct access *access = work_queue_head;
+ struct access *access = rhs_work_queue_head;
- work_queue_head = access->next_queued;
- access->next_queued = NULL;
- access->grp_queued = 0;
+ rhs_work_queue_head = access->next_rhs_queued;
+ access->next_rhs_queued = NULL;
+ access->grp_rhs_queued = 0;
return access;
}
+/* Pop an access from the work queue for propagating from LHS to RHS, and
+ return it, assuming there is one. */
+
+static struct access *
+pop_access_from_lhs_work_queue (void)
+{
+ struct access *access = lhs_work_queue_head;
+
+ lhs_work_queue_head = access->next_lhs_queued;
+ access->next_lhs_queued = NULL;
+ access->grp_lhs_queued = 0;
+ return access;
+}
/* Allocate necessary structures. */
gcc_obstack_init (&name_obstack);
base_access_vec = new hash_map<tree, auto_vec<access_p> >;
memset (&sra_stats, 0, sizeof (sra_stats));
- encountered_apply_args = false;
- encountered_recursive_call = false;
- encountered_unchangable_recursive_call = false;
}
/* Deallocate all general structures. */
}
/* Return true iff the type contains a field or an element which does not allow
- scalarization. */
+ scalarization. Use VISITED_TYPES to avoid re-checking already checked
+ (sub-)types. */
static bool
-type_internals_preclude_sra_p (tree type, const char **msg)
+type_internals_preclude_sra_p_1 (tree type, const char **msg,
+ hash_set<tree> *visited_types)
{
tree fld;
tree et;
+ if (visited_types->contains (type))
+ return false;
+ visited_types->add (type);
+
switch (TREE_CODE (type))
{
case RECORD_TYPE:
for (fld = TYPE_FIELDS (type); fld; fld = DECL_CHAIN (fld))
if (TREE_CODE (fld) == FIELD_DECL)
{
+ if (TREE_CODE (fld) == FUNCTION_DECL)
+ continue;
tree ft = TREE_TYPE (fld);
if (TREE_THIS_VOLATILE (fld))
return true;
}
- if (AGGREGATE_TYPE_P (ft) && type_internals_preclude_sra_p (ft, msg))
+ if (AGGREGATE_TYPE_P (ft)
+ && type_internals_preclude_sra_p_1 (ft, msg, visited_types))
return true;
}
return true;
}
- if (AGGREGATE_TYPE_P (et) && type_internals_preclude_sra_p (et, msg))
+ if (AGGREGATE_TYPE_P (et)
+ && type_internals_preclude_sra_p_1 (et, msg, visited_types))
return true;
return false;
}
}
-/* If T is an SSA_NAME, return NULL if it is not a default def or return its
- base variable if it is. Return T if it is not an SSA_NAME. */
+/* Return true iff the type contains a field or an element which does not allow
+ scalarization. */
-static tree
-get_ssa_base_param (tree t)
+bool
+type_internals_preclude_sra_p (tree type, const char **msg)
{
- if (TREE_CODE (t) == SSA_NAME)
- {
- if (SSA_NAME_IS_DEFAULT_DEF (t))
- return SSA_NAME_VAR (t);
- else
- return NULL_TREE;
- }
- return t;
+ hash_set<tree> visited_types;
+ return type_internals_preclude_sra_p_1 (type, msg, &visited_types);
}
-/* Mark a dereference of BASE of distance DIST in a basic block tht STMT
- belongs to, unless the BB has already been marked as a potentially
- final. */
-
-static void
-mark_parm_dereference (tree base, HOST_WIDE_INT dist, gimple *stmt)
-{
- basic_block bb = gimple_bb (stmt);
- int idx, parm_index = 0;
- tree parm;
-
- if (bitmap_bit_p (final_bbs, bb->index))
- return;
-
- for (parm = DECL_ARGUMENTS (current_function_decl);
- parm && parm != base;
- parm = DECL_CHAIN (parm))
- parm_index++;
-
- gcc_assert (parm_index < func_param_count);
-
- idx = bb->index * func_param_count + parm_index;
- if (bb_dereferences[idx] < dist)
- bb_dereferences[idx] = dist;
-}
/* Allocate an access structure for BASE, OFFSET and SIZE, clear it, fill in
the three fields. Also add it to the vector of accesses corresponding to
{
struct access *access;
poly_int64 poffset, psize, pmax_size;
- HOST_WIDE_INT offset, size, max_size;
tree base = expr;
- bool reverse, ptr, unscalarizable_region = false;
+ bool reverse, unscalarizable_region = false;
base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size,
&reverse);
- if (!poffset.is_constant (&offset)
- || !psize.is_constant (&size)
- || !pmax_size.is_constant (&max_size))
- {
- disqualify_candidate (base, "Encountered a polynomial-sized access.");
- return NULL;
- }
-
- if (sra_mode == SRA_MODE_EARLY_IPA
- && TREE_CODE (base) == MEM_REF)
- {
- base = get_ssa_base_param (TREE_OPERAND (base, 0));
- if (!base)
- return NULL;
- ptr = true;
- }
- else
- ptr = false;
/* For constant-pool entries, check we can substitute the constant value. */
- if (constant_decl_p (base)
- && (sra_mode == SRA_MODE_EARLY_INTRA || sra_mode == SRA_MODE_INTRA))
+ if (constant_decl_p (base))
{
gcc_assert (!bitmap_bit_p (disqualified_constants, DECL_UID (base)));
if (expr != base
if (!DECL_P (base) || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
return NULL;
- if (sra_mode == SRA_MODE_EARLY_IPA)
+ HOST_WIDE_INT offset, size, max_size;
+ if (!poffset.is_constant (&offset)
+ || !psize.is_constant (&size)
+ || !pmax_size.is_constant (&max_size))
{
- if (size < 0 || size != max_size)
- {
- disqualify_candidate (base, "Encountered a variable sized access.");
- return NULL;
- }
- if (TREE_CODE (expr) == COMPONENT_REF
- && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
- {
- disqualify_candidate (base, "Encountered a bit-field access.");
- return NULL;
- }
- gcc_checking_assert ((offset % BITS_PER_UNIT) == 0);
+ disqualify_candidate (base, "Encountered a polynomial-sized access.");
+ return NULL;
+ }
- if (ptr)
- mark_parm_dereference (base, offset + size, stmt);
+ if (size != max_size)
+ {
+ size = max_size;
+ unscalarizable_region = true;
}
- else
+ if (size == 0)
+ return NULL;
+ if (size < 0)
{
- if (size != max_size)
- {
- size = max_size;
- unscalarizable_region = true;
- }
- if (size < 0)
- {
- disqualify_candidate (base, "Encountered an unconstrained access.");
- return NULL;
- }
+ disqualify_candidate (base, "Encountered an unconstrained access.");
+ return NULL;
}
access = create_access_1 (base, offset, size);
access->stmt = stmt;
access->reverse = reverse;
- if (TREE_CODE (expr) == COMPONENT_REF
- && DECL_NONADDRESSABLE_P (TREE_OPERAND (expr, 1)))
- access->non_addressable = 1;
-
return access;
}
static bool
scalarizable_type_p (tree type, bool const_decl)
{
- gcc_assert (!is_gimple_reg_type (type));
+ if (is_gimple_reg_type (type))
+ return true;
if (type_contains_placeholder_p (type))
return false;
+ bool have_predecessor_field = false;
+ HOST_WIDE_INT prev_pos = 0;
+
switch (TREE_CODE (type))
{
case RECORD_TYPE:
{
tree ft = TREE_TYPE (fld);
+ if (zerop (DECL_SIZE (fld)))
+ continue;
+
+ HOST_WIDE_INT pos = int_bit_position (fld);
+ if (have_predecessor_field
+ && pos <= prev_pos)
+ return false;
+
+ have_predecessor_field = true;
+ prev_pos = pos;
+
if (DECL_BIT_FIELD (fld))
return false;
- if (!is_gimple_reg_type (ft)
- && !scalarizable_type_p (ft, const_decl))
+ if (!scalarizable_type_p (ft, const_decl))
return false;
}
return false;
tree elem = TREE_TYPE (type);
- if (!is_gimple_reg_type (elem)
- && !scalarizable_type_p (elem, const_decl))
+ if (!scalarizable_type_p (elem, const_decl))
return false;
return true;
}
}
}
-static void scalarize_elem (tree, HOST_WIDE_INT, HOST_WIDE_INT, bool, tree, tree);
-
-/* Create total_scalarization accesses for all scalar fields of a member
- of type DECL_TYPE conforming to scalarizable_type_p. BASE
- must be the top-most VAR_DECL representing the variable; within that,
- OFFSET locates the member and REF must be the memory reference expression for
- the member. */
-
-static void
-completely_scalarize (tree base, tree decl_type, HOST_WIDE_INT offset, tree ref)
-{
- switch (TREE_CODE (decl_type))
- {
- case RECORD_TYPE:
- for (tree fld = TYPE_FIELDS (decl_type); fld; fld = DECL_CHAIN (fld))
- if (TREE_CODE (fld) == FIELD_DECL)
- {
- HOST_WIDE_INT pos = offset + int_bit_position (fld);
- tree ft = TREE_TYPE (fld);
- tree nref = build3 (COMPONENT_REF, ft, ref, fld, NULL_TREE);
-
- scalarize_elem (base, pos, tree_to_uhwi (DECL_SIZE (fld)),
- TYPE_REVERSE_STORAGE_ORDER (decl_type),
- nref, ft);
- }
- break;
- case ARRAY_TYPE:
- {
- tree elemtype = TREE_TYPE (decl_type);
- tree elem_size = TYPE_SIZE (elemtype);
- gcc_assert (elem_size && tree_fits_shwi_p (elem_size));
- HOST_WIDE_INT el_size = tree_to_shwi (elem_size);
- gcc_assert (el_size > 0);
-
- tree minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (decl_type));
- gcc_assert (TREE_CODE (minidx) == INTEGER_CST);
- tree maxidx = TYPE_MAX_VALUE (TYPE_DOMAIN (decl_type));
- /* Skip (some) zero-length arrays; others have MAXIDX == MINIDX - 1. */
- if (maxidx)
- {
- gcc_assert (TREE_CODE (maxidx) == INTEGER_CST);
- tree domain = TYPE_DOMAIN (decl_type);
- /* MINIDX and MAXIDX are inclusive, and must be interpreted in
- DOMAIN (e.g. signed int, whereas min/max may be size_int). */
- offset_int idx = wi::to_offset (minidx);
- offset_int max = wi::to_offset (maxidx);
- if (!TYPE_UNSIGNED (domain))
- {
- idx = wi::sext (idx, TYPE_PRECISION (domain));
- max = wi::sext (max, TYPE_PRECISION (domain));
- }
- for (int el_off = offset; idx <= max; ++idx)
- {
- tree nref = build4 (ARRAY_REF, elemtype,
- ref,
- wide_int_to_tree (domain, idx),
- NULL_TREE, NULL_TREE);
- scalarize_elem (base, el_off, el_size,
- TYPE_REVERSE_STORAGE_ORDER (decl_type),
- nref, elemtype);
- el_off += el_size;
- }
- }
- }
- break;
- default:
- gcc_unreachable ();
- }
-}
-
-/* Create total_scalarization accesses for a member of type TYPE, which must
- satisfy either is_gimple_reg_type or scalarizable_type_p. BASE must be the
- top-most VAR_DECL representing the variable; within that, POS and SIZE locate
- the member, REVERSE gives its torage order. and REF must be the reference
- expression for it. */
-
-static void
-scalarize_elem (tree base, HOST_WIDE_INT pos, HOST_WIDE_INT size, bool reverse,
- tree ref, tree type)
-{
- if (is_gimple_reg_type (type))
- {
- struct access *access = create_access_1 (base, pos, size);
- access->expr = ref;
- access->type = type;
- access->grp_total_scalarization = 1;
- access->reverse = reverse;
- /* Accesses for intraprocedural SRA can have their stmt NULL. */
- }
- else
- completely_scalarize (base, type, pos, ref);
-}
-
-/* Create a total_scalarization access for VAR as a whole. VAR must be of a
- RECORD_TYPE or ARRAY_TYPE conforming to scalarizable_type_p. */
-
-static void
-create_total_scalarization_access (tree var)
-{
- HOST_WIDE_INT size = tree_to_uhwi (DECL_SIZE (var));
- struct access *access;
-
- access = create_access_1 (var, 0, size);
- access->expr = var;
- access->type = TREE_TYPE (var);
- access->grp_total_scalarization = 1;
-}
-
/* Return true if REF has an VIEW_CONVERT_EXPR somewhere in it. */
static inline bool
return false;
}
-/* Return true if REF contains a VIEW_CONVERT_EXPR or a MEM_REF that performs
- type conversion or a COMPONENT_REF with a bit-field field declaration. */
+/* Return true if REF contains a VIEW_CONVERT_EXPR or a COMPONENT_REF with a
+ bit-field field declaration. If TYPE_CHANGING_P is non-NULL, set the bool
+ it points to will be set if REF contains any of the above or a MEM_REF
+ expression that effectively performs type conversion. */
static bool
-contains_vce_or_bfcref_p (const_tree ref)
+contains_vce_or_bfcref_p (const_tree ref, bool *type_changing_p = NULL)
{
while (handled_component_p (ref))
{
if (TREE_CODE (ref) == VIEW_CONVERT_EXPR
|| (TREE_CODE (ref) == COMPONENT_REF
&& DECL_BIT_FIELD (TREE_OPERAND (ref, 1))))
- return true;
+ {
+ if (type_changing_p)
+ *type_changing_p = true;
+ return true;
+ }
ref = TREE_OPERAND (ref, 0);
}
- if (TREE_CODE (ref) != MEM_REF
+ if (!type_changing_p
+ || TREE_CODE (ref) != MEM_REF
|| TREE_CODE (TREE_OPERAND (ref, 0)) != ADDR_EXPR)
return false;
tree mem = TREE_OPERAND (TREE_OPERAND (ref, 0), 0);
if (TYPE_MAIN_VARIANT (TREE_TYPE (ref))
!= TYPE_MAIN_VARIANT (TREE_TYPE (mem)))
- return true;
+ *type_changing_p = true;
return false;
}
disqualify_base_of_expr (tree t, const char *reason)
{
t = get_base_address (t);
- if (sra_mode == SRA_MODE_EARLY_IPA
- && TREE_CODE (t) == MEM_REF)
- t = get_ssa_base_param (TREE_OPERAND (t, 0));
-
if (t && DECL_P (t))
disqualify_candidate (t, reason);
}
switch (TREE_CODE (expr))
{
case MEM_REF:
- if (TREE_CODE (TREE_OPERAND (expr, 0)) != ADDR_EXPR
- && sra_mode != SRA_MODE_EARLY_IPA)
+ if (TREE_CODE (TREE_OPERAND (expr, 0)) != ADDR_EXPR)
return NULL;
/* fall through */
case VAR_DECL:
static bool
disqualify_if_bad_bb_terminating_stmt (gimple *stmt, tree lhs, tree rhs)
{
- if ((sra_mode == SRA_MODE_EARLY_INTRA || sra_mode == SRA_MODE_INTRA)
- && stmt_ends_bb_p (stmt))
+ if (stmt_ends_bb_p (stmt))
{
if (single_non_eh_succ (gimple_bb (stmt)))
return false;
lacc->grp_assignment_write = 1;
if (storage_order_barrier_p (rhs))
lacc->grp_unscalarizable_region = 1;
+
+ if (should_scalarize_away_bitmap && !is_gimple_reg_type (lacc->type))
+ {
+ bool type_changing_p = false;
+ contains_vce_or_bfcref_p (lhs, &type_changing_p);
+ if (type_changing_p)
+ bitmap_set_bit (cannot_scalarize_away_bitmap,
+ DECL_UID (lacc->base));
+ }
}
if (racc)
{
racc->grp_assignment_read = 1;
- if (should_scalarize_away_bitmap && !gimple_has_volatile_ops (stmt)
- && !is_gimple_reg_type (racc->type))
+ if (should_scalarize_away_bitmap && !is_gimple_reg_type (racc->type))
{
- if (contains_vce_or_bfcref_p (rhs))
+ bool type_changing_p = false;
+ contains_vce_or_bfcref_p (rhs, &type_changing_p);
+
+ if (type_changing_p || gimple_has_volatile_ops (stmt))
bitmap_set_bit (cannot_scalarize_away_bitmap,
DECL_UID (racc->base));
else
link->lacc = lacc;
link->racc = racc;
add_link_to_rhs (racc, link);
- add_access_to_work_queue (racc);
+ add_link_to_lhs (lacc, link);
+ add_access_to_rhs_work_queue (racc);
+ add_access_to_lhs_work_queue (lacc);
/* Let's delay marking the areas as written until propagation of accesses
across link, unless the nature of rhs tells us that its data comes
return false;
}
-/* Return true iff callsite CALL has at least as many actual arguments as there
- are formal parameters of the function currently processed by IPA-SRA and
- that their types match. */
-
-static inline bool
-callsite_arguments_match_p (gimple *call)
-{
- if (gimple_call_num_args (call) < (unsigned) func_param_count)
- return false;
-
- tree parm;
- int i;
- for (parm = DECL_ARGUMENTS (current_function_decl), i = 0;
- parm;
- parm = DECL_CHAIN (parm), i++)
- {
- tree arg = gimple_call_arg (call, i);
- if (!useless_type_conversion_p (TREE_TYPE (parm), TREE_TYPE (arg)))
- return false;
- }
- return true;
-}
-
/* Scan function and look for interesting expressions and create access
structures for them. Return true iff any access is created. */
tree t;
unsigned i;
- if (final_bbs && stmt_can_throw_external (stmt))
- bitmap_set_bit (final_bbs, bb->index);
switch (gimple_code (stmt))
{
case GIMPLE_RETURN:
t = gimple_return_retval (as_a <greturn *> (stmt));
if (t != NULL_TREE)
ret |= build_access_from_expr (t, stmt, false);
- if (final_bbs)
- bitmap_set_bit (final_bbs, bb->index);
break;
case GIMPLE_ASSIGN:
ret |= build_access_from_expr (gimple_call_arg (stmt, i),
stmt, false);
- if (sra_mode == SRA_MODE_EARLY_IPA)
- {
- tree dest = gimple_call_fndecl (stmt);
- int flags = gimple_call_flags (stmt);
-
- if (dest)
- {
- if (DECL_BUILT_IN_CLASS (dest) == BUILT_IN_NORMAL
- && DECL_FUNCTION_CODE (dest) == BUILT_IN_APPLY_ARGS)
- encountered_apply_args = true;
- if (recursive_call_p (current_function_decl, dest))
- {
- encountered_recursive_call = true;
- if (!callsite_arguments_match_p (stmt))
- encountered_unchangable_recursive_call = true;
- }
- }
-
- if (final_bbs
- && (flags & (ECF_CONST | ECF_PURE)) == 0)
- bitmap_set_bit (final_bbs, bb->index);
- }
-
t = gimple_call_lhs (stmt);
if (t && !disqualify_if_bad_bb_terminating_stmt (stmt, t, NULL))
ret |= build_access_from_expr (t, stmt, true);
gasm *asm_stmt = as_a <gasm *> (stmt);
walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
asm_visit_addr);
- if (final_bbs)
- bitmap_set_bit (final_bbs, bb->index);
-
for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
{
t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
return mem_ref;
}
+/* Construct and return a memory reference that is equal to a portion of
+ MODEL->expr but is based on BASE. If this cannot be done, return NULL. */
+
+static tree
+build_reconstructed_reference (location_t, tree base, struct access *model)
+{
+ tree expr = model->expr, prev_expr = NULL;
+ while (!types_compatible_p (TREE_TYPE (expr), TREE_TYPE (base)))
+ {
+ if (!handled_component_p (expr))
+ return NULL_TREE;
+ prev_expr = expr;
+ expr = TREE_OPERAND (expr, 0);
+ }
+
+ /* Guard against broken VIEW_CONVERT_EXPRs... */
+ if (!prev_expr)
+ return NULL_TREE;
+
+ TREE_OPERAND (prev_expr, 0) = base;
+ tree ref = unshare_expr (model->expr);
+ TREE_OPERAND (prev_expr, 0) = expr;
+ return ref;
+}
+
/* Construct a memory reference to a part of an aggregate BASE at the given
OFFSET and of the same type as MODEL. In case this is a reference to a
bit-field, the function will replicate the last component_ref of model's
NULL_TREE);
}
else
- return
- build_ref_for_offset (loc, base, offset, model->reverse, model->type,
- gsi, insert_after);
+ {
+ tree res;
+ if (model->grp_same_access_path
+ && !TREE_THIS_VOLATILE (base)
+ && (TYPE_ADDR_SPACE (TREE_TYPE (base))
+ == TYPE_ADDR_SPACE (TREE_TYPE (model->expr)))
+ && offset <= model->offset
+ /* build_reconstructed_reference can still fail if we have already
+ massaged BASE because of another type incompatibility. */
+ && (res = build_reconstructed_reference (loc, base, model)))
+ return res;
+ else
+ return build_ref_for_offset (loc, base, offset, model->reverse,
+ model->type, gsi, insert_after);
+ }
}
/* Attempt to build a memory reference that we could but into a gimple
}
}
-/* Return true iff TYPE is stdarg va_list type. */
-
-static inline bool
-is_va_list_type (tree type)
-{
- return TYPE_MAIN_VARIANT (type) == TYPE_MAIN_VARIANT (va_list_type_node);
-}
-
/* Print message to dump file why a variable was rejected. */
static void
reject (var, "not aggregate");
return false;
}
- /* Allow constant-pool entries (that "need to live in memory")
- unless we are doing IPA SRA. */
- if (needs_to_live_in_memory (var)
- && (sra_mode == SRA_MODE_EARLY_IPA || !constant_decl_p (var)))
+ /* Allow constant-pool entries that "need to live in memory". */
+ if (needs_to_live_in_memory (var) && !constant_decl_p (var))
{
reject (var, "needs to live in memory");
return false;
return ret;
}
-/* Sort all accesses for the given variable, check for partial overlaps and
- return NULL if there are any. If there are none, pick a representative for
- each combination of offset and size and create a linked list out of them.
- Return the pointer to the first representative and make sure it is the first
- one in the vector of accesses. */
+/* Return true if EXP is a reference chain of COMPONENT_REFs and AREAY_REFs
+ ending either with a DECL or a MEM_REF with zero offset. */
-static struct access *
-sort_and_splice_var_accesses (tree var)
+static bool
+path_comparable_for_same_access (tree expr)
{
- int i, j, access_count;
- struct access *res, **prev_acc_ptr = &res;
- vec<access_p> *access_vec;
- bool first = true;
- HOST_WIDE_INT low = -1, high = 0;
+ while (handled_component_p (expr))
+ {
+ if (TREE_CODE (expr) == ARRAY_REF)
+ {
+ /* SSA name indices can occur here too when the array is of sie one.
+ But we cannot just re-use array_refs with SSA names elsewhere in
+ the function, so disallow non-constant indices. TODO: Remove this
+ limitation after teaching build_reconstructed_reference to replace
+ the index with the index type lower bound. */
+ if (TREE_CODE (TREE_OPERAND (expr, 1)) != INTEGER_CST)
+ return false;
+ }
+ expr = TREE_OPERAND (expr, 0);
+ }
- access_vec = get_base_access_vector (var);
- if (!access_vec)
- return NULL;
- access_count = access_vec->length ();
+ if (TREE_CODE (expr) == MEM_REF)
+ {
+ if (!zerop (TREE_OPERAND (expr, 1)))
+ return false;
+ }
+ else
+ gcc_assert (DECL_P (expr));
+
+ return true;
+}
+
+/* Assuming that EXP1 consists of only COMPONENT_REFs and ARRAY_REFs, return
+ true if the chain of these handled components are exactly the same as EXP2
+ and the expression under them is the same DECL or an equivalent MEM_REF.
+ The reference picked by compare_access_positions must go to EXP1. */
+
+static bool
+same_access_path_p (tree exp1, tree exp2)
+{
+ if (TREE_CODE (exp1) != TREE_CODE (exp2))
+ {
+ /* Special case single-field structures loaded sometimes as the field
+ and sometimes as the structure. If the field is of a scalar type,
+ compare_access_positions will put it into exp1.
+
+ TODO: The gimple register type condition can be removed if teach
+ compare_access_positions to put inner types first. */
+ if (is_gimple_reg_type (TREE_TYPE (exp1))
+ && TREE_CODE (exp1) == COMPONENT_REF
+ && (TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (exp1, 0)))
+ == TYPE_MAIN_VARIANT (TREE_TYPE (exp2))))
+ exp1 = TREE_OPERAND (exp1, 0);
+ else
+ return false;
+ }
+
+ if (!operand_equal_p (exp1, exp2, OEP_ADDRESS_OF))
+ return false;
+
+ return true;
+}
+
+/* Sort all accesses for the given variable, check for partial overlaps and
+ return NULL if there are any. If there are none, pick a representative for
+ each combination of offset and size and create a linked list out of them.
+ Return the pointer to the first representative and make sure it is the first
+ one in the vector of accesses. */
+
+static struct access *
+sort_and_splice_var_accesses (tree var)
+{
+ int i, j, access_count;
+ struct access *res, **prev_acc_ptr = &res;
+ vec<access_p> *access_vec;
+ bool first = true;
+ HOST_WIDE_INT low = -1, high = 0;
+
+ access_vec = get_base_access_vector (var);
+ if (!access_vec)
+ return NULL;
+ access_count = access_vec->length ();
/* Sort by <OFFSET, SIZE>. */
access_vec->qsort (compare_access_positions);
bool grp_assignment_read = access->grp_assignment_read;
bool grp_assignment_write = access->grp_assignment_write;
bool multiple_scalar_reads = false;
- bool total_scalarization = access->grp_total_scalarization;
bool grp_partial_lhs = access->grp_partial_lhs;
bool first_scalar = is_gimple_reg_type (access->type);
bool unscalarizable_region = access->grp_unscalarizable_region;
+ bool grp_same_access_path = true;
bool bf_non_full_precision
= (INTEGRAL_TYPE_P (access->type)
&& TYPE_PRECISION (access->type) != access->size
gcc_assert (access->offset >= low
&& access->offset + access->size <= high);
+ grp_same_access_path = path_comparable_for_same_access (access->expr);
+
j = i + 1;
while (j < access_count)
{
grp_assignment_write |= ac2->grp_assignment_write;
grp_partial_lhs |= ac2->grp_partial_lhs;
unscalarizable_region |= ac2->grp_unscalarizable_region;
- total_scalarization |= ac2->grp_total_scalarization;
relink_to_new_repr (access, ac2);
/* If there are both aggregate-type and scalar-type accesses with
}
unscalarizable_region = true;
}
+
+ if (grp_same_access_path
+ && !same_access_path_p (access->expr, ac2->expr))
+ grp_same_access_path = false;
+
ac2->group_representative = access;
j++;
}
access->grp_scalar_write = grp_scalar_write;
access->grp_assignment_read = grp_assignment_read;
access->grp_assignment_write = grp_assignment_write;
- access->grp_hint = total_scalarization
- || (multiple_scalar_reads && !constant_decl_p (var));
- access->grp_total_scalarization = total_scalarization;
+ access->grp_hint = multiple_scalar_reads && !constant_decl_p (var);
access->grp_partial_lhs = grp_partial_lhs;
access->grp_unscalarizable_region = unscalarizable_region;
+ access->grp_same_access_path = grp_same_access_path;
*prev_acc_ptr = access;
prev_acc_ptr = &access->next_grp;
/* Create a variable for the given ACCESS which determines the type, name and a
few other properties. Return the variable declaration and store it also to
- ACCESS->replacement. */
+ ACCESS->replacement. REG_TREE is used when creating a declaration to base a
+ default-definition SSA name on in order to facilitate an uninitialized
+ warning. It is used instead of the actual ACCESS type if that is not of a
+ gimple register type. */
static tree
-create_access_replacement (struct access *access)
+create_access_replacement (struct access *access, tree reg_type = NULL_TREE)
{
tree repl;
+ tree type = access->type;
+ if (reg_type && !is_gimple_reg_type (type))
+ type = reg_type;
+
if (access->grp_to_be_debug_replaced)
{
repl = create_tmp_var_raw (access->type);
else
/* Drop any special alignment on the type if it's not on the main
variant. This avoids issues with weirdo ABIs like AAPCS. */
- repl = create_tmp_var (build_qualified_type
- (TYPE_MAIN_VARIANT (access->type),
- TYPE_QUALS (access->type)), "SR");
- if (TREE_CODE (access->type) == COMPLEX_TYPE
- || TREE_CODE (access->type) == VECTOR_TYPE)
- {
- if (!access->grp_partial_lhs)
- DECL_GIMPLE_REG_P (repl) = 1;
- }
- else if (access->grp_partial_lhs
- && is_gimple_reg_type (access->type))
- TREE_ADDRESSABLE (repl) = 1;
+ repl = create_tmp_var (build_qualified_type (TYPE_MAIN_VARIANT (type),
+ TYPE_QUALS (type)), "SR");
+ if (access->grp_partial_lhs
+ && is_gimple_reg_type (type))
+ DECL_NOT_GIMPLE_REG_P (repl) = 1;
DECL_SOURCE_LOCATION (repl) = DECL_SOURCE_LOCATION (access->base);
DECL_ARTIFICIAL (repl) = 1;
print_generic_expr (dump_file, access->base);
fprintf (dump_file, " offset: %u, size: %u: ",
(unsigned) access->offset, (unsigned) access->size);
- print_generic_expr (dump_file, repl);
+ print_generic_expr (dump_file, repl, TDF_UID);
fprintf (dump_file, "\n");
}
}
return true;
}
+/* Traverse the access forest where ROOT is the first root and verify that
+ various important invariants hold true. */
+
+DEBUG_FUNCTION void
+verify_sra_access_forest (struct access *root)
+{
+ struct access *access = root;
+ tree first_base = root->base;
+ gcc_assert (DECL_P (first_base));
+ do
+ {
+ gcc_assert (access->base == first_base);
+ if (access->parent)
+ gcc_assert (access->offset >= access->parent->offset
+ && access->size <= access->parent->size);
+ if (access->next_sibling)
+ gcc_assert (access->next_sibling->offset
+ >= access->offset + access->size);
+
+ poly_int64 poffset, psize, pmax_size;
+ bool reverse;
+ tree base = get_ref_base_and_extent (access->expr, &poffset, &psize,
+ &pmax_size, &reverse);
+ HOST_WIDE_INT offset, size, max_size;
+ if (!poffset.is_constant (&offset)
+ || !psize.is_constant (&size)
+ || !pmax_size.is_constant (&max_size))
+ gcc_unreachable ();
+ gcc_assert (base == first_base);
+ gcc_assert (offset == access->offset);
+ gcc_assert (access->grp_unscalarizable_region
+ || access->grp_total_scalarization
+ || size == max_size);
+ gcc_assert (access->grp_unscalarizable_region
+ || !is_gimple_reg_type (access->type)
+ || size == access->size);
+ gcc_assert (reverse == access->reverse);
+
+ if (access->first_child)
+ {
+ gcc_assert (access->first_child->parent == access);
+ access = access->first_child;
+ }
+ else if (access->next_sibling)
+ {
+ gcc_assert (access->next_sibling->parent == access->parent);
+ access = access->next_sibling;
+ }
+ else
+ {
+ while (access->parent && !access->next_sibling)
+ access = access->parent;
+ if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ gcc_assert (access == root);
+ root = root->next_grp;
+ access = root;
+ }
+ }
+ }
+ while (access);
+}
+
+/* Verify access forests of all candidates with accesses by calling
+ verify_access_forest on each on them. */
+
+DEBUG_FUNCTION void
+verify_all_sra_access_forests (void)
+{
+ bitmap_iterator bi;
+ unsigned i;
+ EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
+ {
+ tree var = candidate (i);
+ struct access *access = get_first_repr_for_decl (var);
+ if (access)
+ {
+ gcc_assert (access->base == var);
+ verify_sra_access_forest (access);
+ }
+ }
+}
+
/* Return true if expr contains some ARRAY_REFs into a variable bounded
array. */
}
/* Analyze the subtree of accesses rooted in ROOT, scheduling replacements when
- both seeming beneficial and when ALLOW_REPLACEMENTS allows it. Also set all
- sorts of access flags appropriately along the way, notably always set
- grp_read and grp_assign_read according to MARK_READ and grp_write when
- MARK_WRITE is true.
+ both seeming beneficial and when ALLOW_REPLACEMENTS allows it. If TOTALLY
+ is set, we are totally scalarizing the aggregate. Also set all sorts of
+ access flags appropriately along the way, notably always set grp_read and
+ grp_assign_read according to MARK_READ and grp_write when MARK_WRITE is
+ true.
Creating a replacement for a scalar access is considered beneficial if its
- grp_hint is set (this means we are either attempting total scalarization or
- there is more than one direct read access) or according to the following
- table:
+ grp_hint ot TOTALLY is set (this means either that there is more than one
+ direct read access or that we are attempting total scalarization) or
+ according to the following table:
Access written to through a scalar type (once or more times)
|
static bool
analyze_access_subtree (struct access *root, struct access *parent,
- bool allow_replacements)
+ bool allow_replacements, bool totally)
{
struct access *child;
HOST_WIDE_INT limit = root->offset + root->size;
root->grp_write = 1;
if (parent->grp_assignment_write)
root->grp_assignment_write = 1;
- if (parent->grp_total_scalarization)
- root->grp_total_scalarization = 1;
+ if (!parent->grp_same_access_path)
+ root->grp_same_access_path = 0;
}
if (root->grp_unscalarizable_region)
{
hole |= covered_to < child->offset;
sth_created |= analyze_access_subtree (child, root,
- allow_replacements && !scalar);
+ allow_replacements && !scalar,
+ totally);
root->grp_unscalarized_data |= child->grp_unscalarized_data;
- root->grp_total_scalarization &= child->grp_total_scalarization;
if (child->grp_covered)
covered_to += child->size;
else
}
if (allow_replacements && scalar && !root->first_child
- && (root->grp_hint
+ && (totally || !root->grp_total_scalarization)
+ && (totally
+ || root->grp_hint
|| ((root->grp_scalar_read || root->grp_assignment_read)
&& (root->grp_scalar_write || root->grp_assignment_write))))
{
{
if (allow_replacements
&& scalar && !root->first_child
+ && !root->grp_total_scalarization
&& (root->grp_scalar_write || root->grp_assignment_write)
&& !bitmap_bit_p (cannot_scalarize_away_bitmap,
DECL_UID (root->base)))
root->grp_total_scalarization = 0;
}
- if (!hole || root->grp_total_scalarization)
+ if (!hole || totally)
root->grp_covered = 1;
else if (root->grp_write || comes_initialized_p (root->base))
root->grp_unscalarized_data = 1; /* not covered and written to */
while (access)
{
- if (analyze_access_subtree (access, NULL, true))
+ if (analyze_access_subtree (access, NULL, true,
+ access->grp_total_scalarization))
ret = true;
access = access->next_grp;
}
return ret;
}
-/* Return true iff a potential new child of LACC at offset OFFSET and with size
+/* Return true iff a potential new child of ACC at offset OFFSET and with size
SIZE would conflict with an already existing one. If exactly such a child
- already exists in LACC, store a pointer to it in EXACT_MATCH. */
+ already exists in ACC, store a pointer to it in EXACT_MATCH. */
static bool
-child_would_conflict_in_lacc (struct access *lacc, HOST_WIDE_INT norm_offset,
+child_would_conflict_in_acc (struct access *acc, HOST_WIDE_INT norm_offset,
HOST_WIDE_INT size, struct access **exact_match)
{
struct access *child;
- for (child = lacc->first_child; child; child = child->next_sibling)
+ for (child = acc->first_child; child; child = child->next_sibling)
{
if (child->offset == norm_offset && child->size == size)
{
static struct access *
create_artificial_child_access (struct access *parent, struct access *model,
HOST_WIDE_INT new_offset,
- bool set_grp_write)
+ bool set_grp_read, bool set_grp_write)
{
struct access **child;
tree expr = parent->base;
access->offset = new_offset;
access->size = model->size;
access->type = model->type;
+ access->parent = parent;
+ access->grp_read = set_grp_read;
access->grp_write = set_grp_write;
- access->grp_read = false;
access->reverse = model->reverse;
child = &parent->first_child;
and has assignment links leading from it, re-enqueue it. */
static void
-subtree_mark_written_and_enqueue (struct access *access)
+subtree_mark_written_and_rhs_enqueue (struct access *access)
{
if (access->grp_write)
return;
access->grp_write = true;
- add_access_to_work_queue (access);
+ add_access_to_rhs_work_queue (access);
struct access *child;
for (child = access->first_child; child; child = child->next_sibling)
- subtree_mark_written_and_enqueue (child);
+ subtree_mark_written_and_rhs_enqueue (child);
+}
+
+/* If there is still budget to create a propagation access for DECL, return
+ true and decrement the budget. Otherwise return false. */
+
+static bool
+budget_for_propagation_access (tree decl)
+{
+ unsigned b, *p = propagation_budget->get (decl);
+ if (p)
+ b = *p;
+ else
+ b = param_sra_max_propagations;
+
+ if (b == 0)
+ return false;
+ b--;
+
+ if (b == 0 && dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "The propagation budget of ");
+ print_generic_expr (dump_file, decl);
+ fprintf (dump_file, " (UID: %u) has been exhausted.\n", DECL_UID (decl));
+ }
+ propagation_budget->put (decl, b);
+ return true;
}
/* Propagate subaccesses and grp_write flags of RACC across an assignment link
possible. */
static bool
-propagate_subaccesses_across_link (struct access *lacc, struct access *racc)
+propagate_subaccesses_from_rhs (struct access *lacc, struct access *racc)
{
struct access *rchild;
HOST_WIDE_INT norm_delta = lacc->offset - racc->offset;
gcc_checking_assert (!comes_initialized_p (racc->base));
if (racc->grp_write)
{
- subtree_mark_written_and_enqueue (lacc);
+ subtree_mark_written_and_rhs_enqueue (lacc);
ret = true;
}
}
if (!lacc->grp_write)
{
ret = true;
- subtree_mark_written_and_enqueue (lacc);
+ subtree_mark_written_and_rhs_enqueue (lacc);
}
return ret;
}
if (!lacc->grp_write)
{
ret = true;
- subtree_mark_written_and_enqueue (lacc);
+ subtree_mark_written_and_rhs_enqueue (lacc);
}
if (!lacc->first_child && !racc->first_child)
{
lacc->type = racc->type;
if (build_user_friendly_ref_for_offset (&t, TREE_TYPE (t),
lacc->offset, racc->type))
- lacc->expr = t;
+ {
+ lacc->expr = t;
+ lacc->grp_same_access_path = true;
+ }
else
{
lacc->expr = build_ref_for_model (EXPR_LOCATION (lacc->base),
lacc->base, lacc->offset,
racc, NULL, false);
lacc->grp_no_warning = true;
+ lacc->grp_same_access_path = false;
}
}
return ret;
struct access *new_acc = NULL;
HOST_WIDE_INT norm_offset = rchild->offset + norm_delta;
- if (child_would_conflict_in_lacc (lacc, norm_offset, rchild->size,
+ if (child_would_conflict_in_acc (lacc, norm_offset, rchild->size,
&new_acc))
{
if (new_acc)
if (!new_acc->grp_write && rchild->grp_write)
{
gcc_assert (!lacc->grp_write);
- subtree_mark_written_and_enqueue (new_acc);
+ subtree_mark_written_and_rhs_enqueue (new_acc);
ret = true;
}
rchild->grp_hint = 1;
new_acc->grp_hint |= new_acc->grp_read;
- if (rchild->first_child)
- ret |= propagate_subaccesses_across_link (new_acc, rchild);
+ if (rchild->first_child
+ && propagate_subaccesses_from_rhs (new_acc, rchild))
+ {
+ ret = 1;
+ add_access_to_rhs_work_queue (new_acc);
+ }
}
else
{
if (!lacc->grp_write)
{
ret = true;
- subtree_mark_written_and_enqueue (lacc);
+ subtree_mark_written_and_rhs_enqueue (lacc);
}
}
continue;
}
- if (rchild->grp_unscalarizable_region)
+ if (rchild->grp_unscalarizable_region
+ || !budget_for_propagation_access (lacc->base))
{
if (rchild->grp_write && !lacc->grp_write)
{
ret = true;
- subtree_mark_written_and_enqueue (lacc);
+ subtree_mark_written_and_rhs_enqueue (lacc);
}
continue;
}
rchild->grp_hint = 1;
- new_acc = create_artificial_child_access (lacc, rchild, norm_offset,
- lacc->grp_write
- || rchild->grp_write);
+ /* Because get_ref_base_and_extent always includes padding in size for
+ accesses to DECLs but not necessarily for COMPONENT_REFs of the same
+ type, we might be actually attempting to here to create a child of the
+ same type as the parent. */
+ if (!types_compatible_p (lacc->type, rchild->type))
+ new_acc = create_artificial_child_access (lacc, rchild, norm_offset,
+ false,
+ (lacc->grp_write
+ || rchild->grp_write));
+ else
+ new_acc = lacc;
gcc_checking_assert (new_acc);
if (racc->first_child)
- propagate_subaccesses_across_link (new_acc, rchild);
+ propagate_subaccesses_from_rhs (new_acc, rchild);
- add_access_to_work_queue (lacc);
+ add_access_to_rhs_work_queue (lacc);
ret = true;
}
return ret;
}
+/* Propagate subaccesses of LACC across an assignment link to RACC if they
+ should inhibit total scalarization of the corresponding area. No flags are
+ being propagated in the process. Return true if anything changed. */
+
+static bool
+propagate_subaccesses_from_lhs (struct access *lacc, struct access *racc)
+{
+ if (is_gimple_reg_type (racc->type)
+ || lacc->grp_unscalarizable_region
+ || racc->grp_unscalarizable_region)
+ return false;
+
+ /* TODO: Do we want set some new racc flag to stop potential total
+ scalarization if lacc is a scalar access (and none fo the two have
+ children)? */
+
+ bool ret = false;
+ HOST_WIDE_INT norm_delta = racc->offset - lacc->offset;
+ for (struct access *lchild = lacc->first_child;
+ lchild;
+ lchild = lchild->next_sibling)
+ {
+ struct access *matching_acc = NULL;
+ HOST_WIDE_INT norm_offset = lchild->offset + norm_delta;
+
+ if (lchild->grp_unscalarizable_region
+ || child_would_conflict_in_acc (racc, norm_offset, lchild->size,
+ &matching_acc)
+ || !budget_for_propagation_access (racc->base))
+ {
+ if (matching_acc
+ && propagate_subaccesses_from_lhs (lchild, matching_acc))
+ add_access_to_lhs_work_queue (matching_acc);
+ continue;
+ }
+
+ /* Because get_ref_base_and_extent always includes padding in size for
+ accesses to DECLs but not necessarily for COMPONENT_REFs of the same
+ type, we might be actually attempting to here to create a child of the
+ same type as the parent. */
+ if (!types_compatible_p (racc->type, lchild->type))
+ {
+ struct access *new_acc
+ = create_artificial_child_access (racc, lchild, norm_offset,
+ true, false);
+ propagate_subaccesses_from_lhs (lchild, new_acc);
+ }
+ else
+ propagate_subaccesses_from_lhs (lchild, racc);
+ ret = true;
+ }
+ return ret;
+}
+
/* Propagate all subaccesses across assignment links. */
static void
propagate_all_subaccesses (void)
{
- while (work_queue_head)
+ propagation_budget = new hash_map<tree, unsigned>;
+ while (rhs_work_queue_head)
{
- struct access *racc = pop_access_from_work_queue ();
+ struct access *racc = pop_access_from_rhs_work_queue ();
struct assign_link *link;
if (racc->group_representative)
racc= racc->group_representative;
- gcc_assert (racc->first_link);
+ gcc_assert (racc->first_rhs_link);
- for (link = racc->first_link; link; link = link->next)
+ for (link = racc->first_rhs_link; link; link = link->next_rhs)
{
struct access *lacc = link->lacc;
{
if (!lacc->grp_write)
{
- subtree_mark_written_and_enqueue (lacc);
+ subtree_mark_written_and_rhs_enqueue (lacc);
reque_parents = true;
}
}
- else if (propagate_subaccesses_across_link (lacc, racc))
+ else if (propagate_subaccesses_from_rhs (lacc, racc))
reque_parents = true;
if (reque_parents)
do
{
- add_access_to_work_queue (lacc);
+ add_access_to_rhs_work_queue (lacc);
lacc = lacc->parent;
}
while (lacc);
}
}
+
+ while (lhs_work_queue_head)
+ {
+ struct access *lacc = pop_access_from_lhs_work_queue ();
+ struct assign_link *link;
+
+ if (lacc->group_representative)
+ lacc = lacc->group_representative;
+ gcc_assert (lacc->first_lhs_link);
+
+ if (!bitmap_bit_p (candidate_bitmap, DECL_UID (lacc->base)))
+ continue;
+
+ for (link = lacc->first_lhs_link; link; link = link->next_lhs)
+ {
+ struct access *racc = link->racc;
+
+ if (racc->group_representative)
+ racc = racc->group_representative;
+ if (!bitmap_bit_p (candidate_bitmap, DECL_UID (racc->base)))
+ continue;
+ if (propagate_subaccesses_from_lhs (lacc, racc))
+ add_access_to_lhs_work_queue (racc);
+ }
+ }
+ delete propagation_budget;
}
-/* Go through all accesses collected throughout the (intraprocedural) analysis
- stage, exclude overlapping ones, identify representatives and build trees
- out of them, making decisions about scalarization on the way. Return true
- iff there are any to-be-scalarized variables after this stage. */
+/* Return true if the forest beginning with ROOT does not contain
+ unscalarizable regions or non-byte aligned accesses. */
static bool
-analyze_all_variable_accesses (void)
+can_totally_scalarize_forest_p (struct access *root)
{
- int res = 0;
- bitmap tmp = BITMAP_ALLOC (NULL);
- bitmap_iterator bi;
- unsigned i;
- bool optimize_speed_p = !optimize_function_for_size_p (cfun);
+ struct access *access = root;
+ do
+ {
+ if (access->grp_unscalarizable_region
+ || (access->offset % BITS_PER_UNIT) != 0
+ || (access->size % BITS_PER_UNIT) != 0
+ || (is_gimple_reg_type (access->type)
+ && access->first_child))
+ return false;
- enum compiler_param param = optimize_speed_p
- ? PARAM_SRA_MAX_SCALARIZATION_SIZE_SPEED
- : PARAM_SRA_MAX_SCALARIZATION_SIZE_SIZE;
+ if (access->first_child)
+ access = access->first_child;
+ else if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ while (access->parent && !access->next_sibling)
+ access = access->parent;
+ if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ gcc_assert (access == root);
+ root = root->next_grp;
+ access = root;
+ }
+ }
+ }
+ while (access);
+ return true;
+}
- /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
- fall back to a target default. */
- unsigned HOST_WIDE_INT max_scalarization_size
- = global_options_set.x_param_values[param]
- ? PARAM_VALUE (param)
- : get_move_ratio (optimize_speed_p) * UNITS_PER_WORD;
+/* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
+ reference EXPR for total scalarization purposes and mark it as such. Within
+ the children of PARENT, link it in between PTR and NEXT_SIBLING. */
- max_scalarization_size *= BITS_PER_UNIT;
+static struct access *
+create_total_scalarization_access (struct access *parent, HOST_WIDE_INT pos,
+ HOST_WIDE_INT size, tree type, tree expr,
+ struct access **ptr,
+ struct access *next_sibling)
+{
+ struct access *access = access_pool.allocate ();
+ memset (access, 0, sizeof (struct access));
+ access->base = parent->base;
+ access->offset = pos;
+ access->size = size;
+ access->expr = expr;
+ access->type = type;
+ access->parent = parent;
+ access->grp_write = parent->grp_write;
+ access->grp_total_scalarization = 1;
+ access->grp_hint = 1;
+ access->grp_same_access_path = path_comparable_for_same_access (expr);
+ access->reverse = reverse_storage_order_for_component_p (expr);
- EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
- if (bitmap_bit_p (should_scalarize_away_bitmap, i)
- && !bitmap_bit_p (cannot_scalarize_away_bitmap, i))
- {
- tree var = candidate (i);
+ access->next_sibling = next_sibling;
+ *ptr = access;
+ return access;
+}
- if (VAR_P (var) && scalarizable_type_p (TREE_TYPE (var),
- constant_decl_p (var)))
- {
- if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var)))
- <= max_scalarization_size)
- {
- create_total_scalarization_access (var);
- completely_scalarize (var, TREE_TYPE (var), 0, var);
- statistics_counter_event (cfun,
- "Totally-scalarized aggregates", 1);
- if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Will attempt to totally scalarize ");
- print_generic_expr (dump_file, var);
- fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
- }
- }
- else if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Too big to totally scalarize: ");
- print_generic_expr (dump_file, var);
- fprintf (dump_file, " (UID: %u)\n", DECL_UID (var));
- }
- }
- }
+/* Create and return an ACCESS in PARENT spanning from POS with SIZE, TYPE and
+ reference EXPR for total scalarization purposes and mark it as such, link it
+ at *PTR and reshape the tree so that those elements at *PTR and their
+ siblings which fall within the part described by POS and SIZE are moved to
+ be children of the new access. If a partial overlap is detected, return
+ NULL. */
- bitmap_copy (tmp, candidate_bitmap);
- EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
+static struct access *
+create_total_access_and_reshape (struct access *parent, HOST_WIDE_INT pos,
+ HOST_WIDE_INT size, tree type, tree expr,
+ struct access **ptr)
+{
+ struct access **p = ptr;
+
+ while (*p && (*p)->offset < pos + size)
{
- tree var = candidate (i);
- struct access *access;
+ if ((*p)->offset + (*p)->size > pos + size)
+ return NULL;
+ p = &(*p)->next_sibling;
+ }
- access = sort_and_splice_var_accesses (var);
- if (!access || !build_access_trees (access))
- disqualify_candidate (var,
- "No or inhibitingly overlapping accesses.");
+ struct access *next_child = *ptr;
+ struct access *new_acc
+ = create_total_scalarization_access (parent, pos, size, type, expr,
+ ptr, *p);
+ if (p != ptr)
+ {
+ new_acc->first_child = next_child;
+ *p = NULL;
+ for (struct access *a = next_child; a; a = a->next_sibling)
+ a->parent = new_acc;
}
+ return new_acc;
+}
- propagate_all_subaccesses ();
+static bool totally_scalarize_subtree (struct access *root);
- bitmap_copy (tmp, candidate_bitmap);
- EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
+/* Return true if INNER is either the same type as OUTER or if it is the type
+ of a record field in OUTER at offset zero, possibly in nested
+ sub-records. */
+
+static bool
+access_and_field_type_match_p (tree outer, tree inner)
+{
+ if (TYPE_MAIN_VARIANT (outer) == TYPE_MAIN_VARIANT (inner))
+ return true;
+ if (TREE_CODE (outer) != RECORD_TYPE)
+ return false;
+ tree fld = TYPE_FIELDS (outer);
+ while (fld)
{
- tree var = candidate (i);
- struct access *access = get_first_repr_for_decl (var);
+ if (TREE_CODE (fld) == FIELD_DECL)
+ {
+ if (!zerop (DECL_FIELD_OFFSET (fld)))
+ return false;
+ if (TYPE_MAIN_VARIANT (TREE_TYPE (fld)) == inner)
+ return true;
+ if (TREE_CODE (TREE_TYPE (fld)) == RECORD_TYPE)
+ fld = TYPE_FIELDS (TREE_TYPE (fld));
+ else
+ return false;
+ }
+ else
+ fld = DECL_CHAIN (fld);
+ }
+ return false;
+}
- if (analyze_access_trees (access))
- {
- res++;
- if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "\nAccess trees for ");
- print_generic_expr (dump_file, var);
- fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
- dump_access_tree (dump_file, access);
- fprintf (dump_file, "\n");
- }
- }
- else
- disqualify_candidate (var, "No scalar replacements to be created.");
+/* Return type of total_should_skip_creating_access indicating whether a total
+ scalarization access for a field/element should be created, whether it
+ already exists or whether the entire total scalarization has to fail. */
+
+enum total_sra_field_state {TOTAL_FLD_CREATE, TOTAL_FLD_DONE, TOTAL_FLD_FAILED};
+
+/* Do all the necessary steps in total scalarization when the given aggregate
+ type has a TYPE at POS with the given SIZE should be put into PARENT and
+ when we have processed all its siblings with smaller offsets up until and
+ including LAST_SEEN_SIBLING (which can be NULL).
+
+ If some further siblings are to be skipped, set *LAST_SEEN_SIBLING as
+ appropriate. Return TOTAL_FLD_CREATE id the caller should carry on with
+ creating a new access, TOTAL_FLD_DONE if access or accesses capable of
+ representing the described part of the aggregate for the purposes of total
+ scalarization already exist or TOTAL_FLD_FAILED if there is a problem which
+ prevents total scalarization from happening at all. */
+
+static enum total_sra_field_state
+total_should_skip_creating_access (struct access *parent,
+ struct access **last_seen_sibling,
+ tree type, HOST_WIDE_INT pos,
+ HOST_WIDE_INT size)
+{
+ struct access *next_child;
+ if (!*last_seen_sibling)
+ next_child = parent->first_child;
+ else
+ next_child = (*last_seen_sibling)->next_sibling;
+
+ /* First, traverse the chain of siblings until it points to an access with
+ offset at least equal to POS. Check all skipped accesses whether they
+ span the POS boundary and if so, return with a failure. */
+ while (next_child && next_child->offset < pos)
+ {
+ if (next_child->offset + next_child->size > pos)
+ return TOTAL_FLD_FAILED;
+ *last_seen_sibling = next_child;
+ next_child = next_child->next_sibling;
}
- BITMAP_FREE (tmp);
+ /* Now check whether next_child has exactly the right POS and SIZE and if so,
+ whether it can represent what we need and can be totally scalarized
+ itself. */
+ if (next_child && next_child->offset == pos
+ && next_child->size == size)
+ {
+ if (!is_gimple_reg_type (next_child->type)
+ && (!access_and_field_type_match_p (type, next_child->type)
+ || !totally_scalarize_subtree (next_child)))
+ return TOTAL_FLD_FAILED;
- if (res)
+ *last_seen_sibling = next_child;
+ return TOTAL_FLD_DONE;
+ }
+
+ /* If the child we're looking at would partially overlap, we just cannot
+ totally scalarize. */
+ if (next_child
+ && next_child->offset < pos + size
+ && next_child->offset + next_child->size > pos + size)
+ return TOTAL_FLD_FAILED;
+
+ if (is_gimple_reg_type (type))
{
- statistics_counter_event (cfun, "Scalarized aggregates", res);
- return true;
+ /* We don't scalarize accesses that are children of other scalar type
+ accesses, so if we go on and create an access for a register type,
+ there should not be any pre-existing children. There are rare cases
+ where the requested type is a vector but we already have register
+ accesses for all its elements which is equally good. Detect that
+ situation or whether we need to bail out. */
+
+ HOST_WIDE_INT covered = pos;
+ bool skipping = false;
+ while (next_child
+ && next_child->offset + next_child->size <= pos + size)
+ {
+ if (next_child->offset != covered
+ || !is_gimple_reg_type (next_child->type))
+ return TOTAL_FLD_FAILED;
+
+ covered += next_child->size;
+ *last_seen_sibling = next_child;
+ next_child = next_child->next_sibling;
+ skipping = true;
+ }
+
+ if (skipping)
+ {
+ if (covered != pos + size)
+ return TOTAL_FLD_FAILED;
+ else
+ return TOTAL_FLD_DONE;
+ }
}
- else
- return false;
+
+ return TOTAL_FLD_CREATE;
}
-/* Generate statements copying scalar replacements of accesses within a subtree
- into or out of AGG. ACCESS, all its children, siblings and their children
+/* Go over sub-tree rooted in ROOT and attempt to create scalar accesses
+ spanning all uncovered areas covered by ROOT, return false if the attempt
+ failed. All created accesses will have grp_unscalarizable_region set (and
+ should be ignored if the function returns false). */
+
+static bool
+totally_scalarize_subtree (struct access *root)
+{
+ gcc_checking_assert (!root->grp_unscalarizable_region);
+ gcc_checking_assert (!is_gimple_reg_type (root->type));
+
+ struct access *last_seen_sibling = NULL;
+
+ switch (TREE_CODE (root->type))
+ {
+ case RECORD_TYPE:
+ for (tree fld = TYPE_FIELDS (root->type); fld; fld = DECL_CHAIN (fld))
+ if (TREE_CODE (fld) == FIELD_DECL)
+ {
+ tree ft = TREE_TYPE (fld);
+ HOST_WIDE_INT fsize = tree_to_uhwi (DECL_SIZE (fld));
+ if (!fsize)
+ continue;
+
+ HOST_WIDE_INT pos = root->offset + int_bit_position (fld);
+ enum total_sra_field_state
+ state = total_should_skip_creating_access (root,
+ &last_seen_sibling,
+ ft, pos, fsize);
+ switch (state)
+ {
+ case TOTAL_FLD_FAILED:
+ return false;
+ case TOTAL_FLD_DONE:
+ continue;
+ case TOTAL_FLD_CREATE:
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ struct access **p = (last_seen_sibling
+ ? &last_seen_sibling->next_sibling
+ : &root->first_child);
+ tree nref = build3 (COMPONENT_REF, ft, root->expr, fld, NULL_TREE);
+ struct access *new_child
+ = create_total_access_and_reshape (root, pos, fsize, ft, nref, p);
+ if (!new_child)
+ return false;
+
+ if (!is_gimple_reg_type (ft)
+ && !totally_scalarize_subtree (new_child))
+ return false;
+ last_seen_sibling = new_child;
+ }
+ break;
+ case ARRAY_TYPE:
+ {
+ tree elemtype = TREE_TYPE (root->type);
+ tree elem_size = TYPE_SIZE (elemtype);
+ gcc_assert (elem_size && tree_fits_shwi_p (elem_size));
+ HOST_WIDE_INT el_size = tree_to_shwi (elem_size);
+ gcc_assert (el_size > 0);
+
+ tree minidx = TYPE_MIN_VALUE (TYPE_DOMAIN (root->type));
+ gcc_assert (TREE_CODE (minidx) == INTEGER_CST);
+ tree maxidx = TYPE_MAX_VALUE (TYPE_DOMAIN (root->type));
+ /* Skip (some) zero-length arrays; others have MAXIDX == MINIDX - 1. */
+ if (!maxidx)
+ goto out;
+ gcc_assert (TREE_CODE (maxidx) == INTEGER_CST);
+ tree domain = TYPE_DOMAIN (root->type);
+ /* MINIDX and MAXIDX are inclusive, and must be interpreted in
+ DOMAIN (e.g. signed int, whereas min/max may be size_int). */
+ offset_int idx = wi::to_offset (minidx);
+ offset_int max = wi::to_offset (maxidx);
+ if (!TYPE_UNSIGNED (domain))
+ {
+ idx = wi::sext (idx, TYPE_PRECISION (domain));
+ max = wi::sext (max, TYPE_PRECISION (domain));
+ }
+ for (HOST_WIDE_INT pos = root->offset;
+ idx <= max;
+ pos += el_size, ++idx)
+ {
+ enum total_sra_field_state
+ state = total_should_skip_creating_access (root,
+ &last_seen_sibling,
+ elemtype, pos,
+ el_size);
+ switch (state)
+ {
+ case TOTAL_FLD_FAILED:
+ return false;
+ case TOTAL_FLD_DONE:
+ continue;
+ case TOTAL_FLD_CREATE:
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ struct access **p = (last_seen_sibling
+ ? &last_seen_sibling->next_sibling
+ : &root->first_child);
+ tree nref = build4 (ARRAY_REF, elemtype, root->expr,
+ wide_int_to_tree (domain, idx),
+ NULL_TREE, NULL_TREE);
+ struct access *new_child
+ = create_total_access_and_reshape (root, pos, el_size, elemtype,
+ nref, p);
+ if (!new_child)
+ return false;
+
+ if (!is_gimple_reg_type (elemtype)
+ && !totally_scalarize_subtree (new_child))
+ return false;
+ last_seen_sibling = new_child;
+ }
+ }
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ out:
+ return true;
+}
+
+/* Go through all accesses collected throughout the (intraprocedural) analysis
+ stage, exclude overlapping ones, identify representatives and build trees
+ out of them, making decisions about scalarization on the way. Return true
+ iff there are any to-be-scalarized variables after this stage. */
+
+static bool
+analyze_all_variable_accesses (void)
+{
+ int res = 0;
+ bitmap tmp = BITMAP_ALLOC (NULL);
+ bitmap_iterator bi;
+ unsigned i;
+
+ bitmap_copy (tmp, candidate_bitmap);
+ EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
+ {
+ tree var = candidate (i);
+ struct access *access;
+
+ access = sort_and_splice_var_accesses (var);
+ if (!access || !build_access_trees (access))
+ disqualify_candidate (var,
+ "No or inhibitingly overlapping accesses.");
+ }
+
+ propagate_all_subaccesses ();
+
+ bool optimize_speed_p = !optimize_function_for_size_p (cfun);
+ /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
+ fall back to a target default. */
+ unsigned HOST_WIDE_INT max_scalarization_size
+ = get_move_ratio (optimize_speed_p) * UNITS_PER_WORD;
+
+ if (optimize_speed_p)
+ {
+ if (global_options_set.x_param_sra_max_scalarization_size_speed)
+ max_scalarization_size = param_sra_max_scalarization_size_speed;
+ }
+ else
+ {
+ if (global_options_set.x_param_sra_max_scalarization_size_size)
+ max_scalarization_size = param_sra_max_scalarization_size_size;
+ }
+ max_scalarization_size *= BITS_PER_UNIT;
+
+ EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap, 0, i, bi)
+ if (bitmap_bit_p (should_scalarize_away_bitmap, i)
+ && !bitmap_bit_p (cannot_scalarize_away_bitmap, i))
+ {
+ tree var = candidate (i);
+ if (!VAR_P (var))
+ continue;
+
+ if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))) > max_scalarization_size)
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Too big to totally scalarize: ");
+ print_generic_expr (dump_file, var);
+ fprintf (dump_file, " (UID: %u)\n", DECL_UID (var));
+ }
+ continue;
+ }
+
+ bool all_types_ok = true;
+ for (struct access *access = get_first_repr_for_decl (var);
+ access;
+ access = access->next_grp)
+ if (!can_totally_scalarize_forest_p (access)
+ || !scalarizable_type_p (access->type, constant_decl_p (var)))
+ {
+ all_types_ok = false;
+ break;
+ }
+ if (!all_types_ok)
+ continue;
+
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Will attempt to totally scalarize ");
+ print_generic_expr (dump_file, var);
+ fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
+ }
+ bool scalarized = true;
+ for (struct access *access = get_first_repr_for_decl (var);
+ access;
+ access = access->next_grp)
+ if (!is_gimple_reg_type (access->type)
+ && !totally_scalarize_subtree (access))
+ {
+ scalarized = false;
+ break;
+ }
+
+ if (scalarized)
+ for (struct access *access = get_first_repr_for_decl (var);
+ access;
+ access = access->next_grp)
+ access->grp_total_scalarization = true;
+ }
+
+ if (flag_checking)
+ verify_all_sra_access_forests ();
+
+ bitmap_copy (tmp, candidate_bitmap);
+ EXECUTE_IF_SET_IN_BITMAP (tmp, 0, i, bi)
+ {
+ tree var = candidate (i);
+ struct access *access = get_first_repr_for_decl (var);
+
+ if (analyze_access_trees (access))
+ {
+ res++;
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "\nAccess trees for ");
+ print_generic_expr (dump_file, var);
+ fprintf (dump_file, " (UID: %u): \n", DECL_UID (var));
+ dump_access_tree (dump_file, access);
+ fprintf (dump_file, "\n");
+ }
+ }
+ else
+ disqualify_candidate (var, "No scalar replacements to be created.");
+ }
+
+ BITMAP_FREE (tmp);
+
+ if (res)
+ {
+ statistics_counter_event (cfun, "Scalarized aggregates", res);
+ return true;
+ }
+ else
+ return false;
+}
+
+/* Generate statements copying scalar replacements of accesses within a subtree
+ into or out of AGG. ACCESS, all its children, siblings and their children
are to be processed. AGG is an aggregate type expression (can be a
declaration but does not have to be, it can for example also be a mem_ref or
a series of handled components). TOP_OFFSET is the offset of the processed
if (access->grp_to_be_replaced)
{
tree rep = get_access_replacement (access);
- tree clobber = build_constructor (access->type, NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
+ tree clobber = build_clobber (access->type);
gimple *stmt = gimple_build_assign (rep, clobber);
if (insert_after)
|| !DECL_P (base))
return NULL;
- if (!bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
+ if (tree basesize = DECL_SIZE (base))
+ {
+ poly_int64 sz;
+ if (offset < 0
+ || !poly_int_tree_p (basesize, &sz)
+ || known_le (sz, offset))
+ return NULL;
+ }
+
+ if (max_size == 0
+ || !bitmap_bit_p (candidate_bitmap, DECL_UID (base)))
return NULL;
return get_var_base_offset_size_access (base, offset, max_size);
location_t loc;
struct access *access;
tree type, bfr, orig_expr;
+ bool partial_cplx_access = false;
if (TREE_CODE (*expr) == BIT_FIELD_REF)
{
bfr = NULL_TREE;
if (TREE_CODE (*expr) == REALPART_EXPR || TREE_CODE (*expr) == IMAGPART_EXPR)
- expr = &TREE_OPERAND (*expr, 0);
+ {
+ expr = &TREE_OPERAND (*expr, 0);
+ partial_cplx_access = true;
+ }
access = get_access_for_expr (*expr);
if (!access)
return false;
be accessed as a different type too, potentially creating a need for
type conversion (see PR42196) and when scalarized unions are involved
in assembler statements (see PR42398). */
- if (!useless_type_conversion_p (type, access->type))
+ if (!bfr && !useless_type_conversion_p (type, access->type))
{
tree ref;
ref = build_ref_for_model (loc, orig_expr, 0, access, gsi, false);
- if (write)
+ if (partial_cplx_access)
+ {
+ /* VIEW_CONVERT_EXPRs in partial complex access are always fine in
+ the case of a write because in such case the replacement cannot
+ be a gimple register. In the case of a load, we have to
+ differentiate in between a register an non-register
+ replacement. */
+ tree t = build1 (VIEW_CONVERT_EXPR, type, repl);
+ gcc_checking_assert (!write || access->grp_partial_lhs);
+ if (!access->grp_partial_lhs)
+ {
+ tree tmp = make_ssa_name (type);
+ gassign *stmt = gimple_build_assign (tmp, t);
+ /* This is always a read. */
+ gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
+ t = tmp;
+ }
+ *expr = t;
+ }
+ else if (write)
{
gassign *stmt;
/* Create and return a new suitable default definition SSA_NAME for RACC which
is an access describing an uninitialized part of an aggregate that is being
- loaded. */
+ loaded. REG_TREE is used instead of the actual RACC type if that is not of
+ a gimple register type. */
static tree
-get_repl_default_def_ssa_name (struct access *racc)
+get_repl_default_def_ssa_name (struct access *racc, tree reg_type)
{
gcc_checking_assert (!racc->grp_to_be_replaced
&& !racc->grp_to_be_debug_replaced);
if (!racc->replacement_decl)
- racc->replacement_decl = create_access_replacement (racc);
+ racc->replacement_decl = create_access_replacement (racc, reg_type);
return get_or_create_ssa_default_def (cfun, racc->replacement_decl);
}
&& TREE_CODE (lhs) == SSA_NAME
&& !access_has_replacements_p (racc))
{
- rhs = get_repl_default_def_ssa_name (racc);
+ rhs = get_repl_default_def_ssa_name (racc, TREE_TYPE (lhs));
modify_this_stmt = true;
sra_stats.exprs++;
}
lhs = build_ref_for_model (loc, lhs, 0, racc, gsi, false);
gimple_assign_set_lhs (stmt, lhs);
}
- else if (AGGREGATE_TYPE_P (TREE_TYPE (rhs))
+ else if (lacc
+ && AGGREGATE_TYPE_P (TREE_TYPE (rhs))
&& !contains_vce_or_bfcref_p (rhs))
rhs = build_ref_for_model (loc, rhs, 0, lacc, gsi, false);
tree var = candidate (i);
if (!constant_decl_p (var))
continue;
- vec<access_p> *access_vec = get_base_access_vector (var);
- if (!access_vec)
- continue;
- for (unsigned i = 0; i < access_vec->length (); i++)
+
+ struct access *access = get_first_repr_for_decl (var);
+
+ while (access)
{
- struct access *access = (*access_vec)[i];
- if (!access->replacement_decl)
- continue;
- gassign *stmt
- = gimple_build_assign (get_access_replacement (access),
- unshare_expr (access->expr));
- if (dump_file && (dump_flags & TDF_DETAILS))
+ if (access->replacement_decl)
{
- fprintf (dump_file, "Generating constant initializer: ");
- print_gimple_stmt (dump_file, stmt, 0);
- fprintf (dump_file, "\n");
+ gassign *stmt
+ = gimple_build_assign (get_access_replacement (access),
+ unshare_expr (access->expr));
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ {
+ fprintf (dump_file, "Generating constant initializer: ");
+ print_gimple_stmt (dump_file, stmt, 0);
+ fprintf (dump_file, "\n");
+ }
+ gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
+ update_stmt (stmt);
+ }
+
+ if (access->first_child)
+ access = access->first_child;
+ else if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ {
+ while (access->parent && !access->next_sibling)
+ access = access->parent;
+ if (access->next_sibling)
+ access = access->next_sibling;
+ else
+ access = access->next_grp;
}
- gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
- update_stmt (stmt);
}
}
{
return new pass_sra (ctxt);
}
-
-
-/* Return true iff PARM (which must be a parm_decl) is an unused scalar
- parameter. */
-
-static bool
-is_unused_scalar_param (tree parm)
-{
- tree name;
- return (is_gimple_reg (parm)
- && (!(name = ssa_default_def (cfun, parm))
- || has_zero_uses (name)));
-}
-
-/* Scan immediate uses of a default definition SSA name of a parameter PARM and
- examine whether there are any direct or otherwise infeasible ones. If so,
- return true, otherwise return false. PARM must be a gimple register with a
- non-NULL default definition. */
-
-static bool
-ptr_parm_has_direct_uses (tree parm)
-{
- imm_use_iterator ui;
- gimple *stmt;
- tree name = ssa_default_def (cfun, parm);
- bool ret = false;
-
- FOR_EACH_IMM_USE_STMT (stmt, ui, name)
- {
- int uses_ok = 0;
- use_operand_p use_p;
-
- if (is_gimple_debug (stmt))
- continue;
-
- /* Valid uses include dereferences on the lhs and the rhs. */
- if (gimple_has_lhs (stmt))
- {
- tree lhs = gimple_get_lhs (stmt);
- while (handled_component_p (lhs))
- lhs = TREE_OPERAND (lhs, 0);
- if (TREE_CODE (lhs) == MEM_REF
- && TREE_OPERAND (lhs, 0) == name
- && integer_zerop (TREE_OPERAND (lhs, 1))
- && types_compatible_p (TREE_TYPE (lhs),
- TREE_TYPE (TREE_TYPE (name)))
- && !TREE_THIS_VOLATILE (lhs))
- uses_ok++;
- }
- if (gimple_assign_single_p (stmt))
- {
- tree rhs = gimple_assign_rhs1 (stmt);
- while (handled_component_p (rhs))
- rhs = TREE_OPERAND (rhs, 0);
- if (TREE_CODE (rhs) == MEM_REF
- && TREE_OPERAND (rhs, 0) == name
- && integer_zerop (TREE_OPERAND (rhs, 1))
- && types_compatible_p (TREE_TYPE (rhs),
- TREE_TYPE (TREE_TYPE (name)))
- && !TREE_THIS_VOLATILE (rhs))
- uses_ok++;
- }
- else if (is_gimple_call (stmt))
- {
- unsigned i;
- for (i = 0; i < gimple_call_num_args (stmt); ++i)
- {
- tree arg = gimple_call_arg (stmt, i);
- while (handled_component_p (arg))
- arg = TREE_OPERAND (arg, 0);
- if (TREE_CODE (arg) == MEM_REF
- && TREE_OPERAND (arg, 0) == name
- && integer_zerop (TREE_OPERAND (arg, 1))
- && types_compatible_p (TREE_TYPE (arg),
- TREE_TYPE (TREE_TYPE (name)))
- && !TREE_THIS_VOLATILE (arg))
- uses_ok++;
- }
- }
-
- /* If the number of valid uses does not match the number of
- uses in this stmt there is an unhandled use. */
- FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
- --uses_ok;
-
- if (uses_ok != 0)
- ret = true;
-
- if (ret)
- BREAK_FROM_IMM_USE_STMT (ui);
- }
-
- return ret;
-}
-
-/* Identify candidates for reduction for IPA-SRA based on their type and mark
- them in candidate_bitmap. Note that these do not necessarily include
- parameter which are unused and thus can be removed. Return true iff any
- such candidate has been found. */
-
-static bool
-find_param_candidates (void)
-{
- tree parm;
- int count = 0;
- bool ret = false;
- const char *msg;
-
- for (parm = DECL_ARGUMENTS (current_function_decl);
- parm;
- parm = DECL_CHAIN (parm))
- {
- tree type = TREE_TYPE (parm);
- tree_node **slot;
-
- count++;
-
- if (TREE_THIS_VOLATILE (parm)
- || TREE_ADDRESSABLE (parm)
- || (!is_gimple_reg_type (type) && is_va_list_type (type)))
- continue;
-
- if (is_unused_scalar_param (parm))
- {
- ret = true;
- continue;
- }
-
- if (POINTER_TYPE_P (type))
- {
- type = TREE_TYPE (type);
-
- if (TREE_CODE (type) == FUNCTION_TYPE
- || TYPE_VOLATILE (type)
- || (TREE_CODE (type) == ARRAY_TYPE
- && TYPE_NONALIASED_COMPONENT (type))
- || !is_gimple_reg (parm)
- || is_va_list_type (type)
- || ptr_parm_has_direct_uses (parm))
- continue;
- }
- else if (!AGGREGATE_TYPE_P (type))
- continue;
-
- if (!COMPLETE_TYPE_P (type)
- || !tree_fits_uhwi_p (TYPE_SIZE (type))
- || tree_to_uhwi (TYPE_SIZE (type)) == 0
- || (AGGREGATE_TYPE_P (type)
- && type_internals_preclude_sra_p (type, &msg)))
- continue;
-
- bitmap_set_bit (candidate_bitmap, DECL_UID (parm));
- slot = candidates->find_slot_with_hash (parm, DECL_UID (parm), INSERT);
- *slot = parm;
-
- ret = true;
- if (dump_file && (dump_flags & TDF_DETAILS))
- {
- fprintf (dump_file, "Candidate (%d): ", DECL_UID (parm));
- print_generic_expr (dump_file, parm);
- fprintf (dump_file, "\n");
- }
- }
-
- func_param_count = count;
- return ret;
-}
-
-/* Callback of walk_aliased_vdefs, marks the access passed as DATA as
- maybe_modified. */
-
-static bool
-mark_maybe_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
- void *data)
-{
- struct access *repr = (struct access *) data;
-
- repr->grp_maybe_modified = 1;
- return true;
-}
-
-/* Analyze what representatives (in linked lists accessible from
- REPRESENTATIVES) can be modified by side effects of statements in the
- current function. */
-
-static void
-analyze_modified_params (vec<access_p> representatives)
-{
- int i;
-
- for (i = 0; i < func_param_count; i++)
- {
- struct access *repr;
-
- for (repr = representatives[i];
- repr;
- repr = repr->next_grp)
- {
- struct access *access;
- bitmap visited;
- ao_ref ar;
-
- if (no_accesses_p (repr))
- continue;
- if (!POINTER_TYPE_P (TREE_TYPE (repr->base))
- || repr->grp_maybe_modified)
- continue;
-
- ao_ref_init (&ar, repr->expr);
- visited = BITMAP_ALLOC (NULL);
- for (access = repr; access; access = access->next_sibling)
- {
- /* All accesses are read ones, otherwise grp_maybe_modified would
- be trivially set. */
- walk_aliased_vdefs (&ar, gimple_vuse (access->stmt),
- mark_maybe_modified, repr, &visited);
- if (repr->grp_maybe_modified)
- break;
- }
- BITMAP_FREE (visited);
- }
- }
-}
-
-/* Propagate distances in bb_dereferences in the opposite direction than the
- control flow edges, in each step storing the maximum of the current value
- and the minimum of all successors. These steps are repeated until the table
- stabilizes. Note that BBs which might terminate the functions (according to
- final_bbs bitmap) never updated in this way. */
-
-static void
-propagate_dereference_distances (void)
-{
- basic_block bb;
-
- auto_vec<basic_block> queue (last_basic_block_for_fn (cfun));
- queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (cfun));
- FOR_EACH_BB_FN (bb, cfun)
- {
- queue.quick_push (bb);
- bb->aux = bb;
- }
-
- while (!queue.is_empty ())
- {
- edge_iterator ei;
- edge e;
- bool change = false;
- int i;
-
- bb = queue.pop ();
- bb->aux = NULL;
-
- if (bitmap_bit_p (final_bbs, bb->index))
- continue;
-
- for (i = 0; i < func_param_count; i++)
- {
- int idx = bb->index * func_param_count + i;
- bool first = true;
- HOST_WIDE_INT inh = 0;
-
- FOR_EACH_EDGE (e, ei, bb->succs)
- {
- int succ_idx = e->dest->index * func_param_count + i;
-
- if (e->src == EXIT_BLOCK_PTR_FOR_FN (cfun))
- continue;
-
- if (first)
- {
- first = false;
- inh = bb_dereferences [succ_idx];
- }
- else if (bb_dereferences [succ_idx] < inh)
- inh = bb_dereferences [succ_idx];
- }
-
- if (!first && bb_dereferences[idx] < inh)
- {
- bb_dereferences[idx] = inh;
- change = true;
- }
- }
-
- if (change && !bitmap_bit_p (final_bbs, bb->index))
- FOR_EACH_EDGE (e, ei, bb->preds)
- {
- if (e->src->aux)
- continue;
-
- e->src->aux = e->src;
- queue.quick_push (e->src);
- }
- }
-}
-
-/* Dump a dereferences TABLE with heading STR to file F. */
-
-static void
-dump_dereferences_table (FILE *f, const char *str, HOST_WIDE_INT *table)
-{
- basic_block bb;
-
- fprintf (dump_file, "%s", str);
- FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun),
- EXIT_BLOCK_PTR_FOR_FN (cfun), next_bb)
- {
- fprintf (f, "%4i %i ", bb->index, bitmap_bit_p (final_bbs, bb->index));
- if (bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
- {
- int i;
- for (i = 0; i < func_param_count; i++)
- {
- int idx = bb->index * func_param_count + i;
- fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", table[idx]);
- }
- }
- fprintf (f, "\n");
- }
- fprintf (dump_file, "\n");
-}
-
-/* Determine what (parts of) parameters passed by reference that are not
- assigned to are not certainly dereferenced in this function and thus the
- dereferencing cannot be safely moved to the caller without potentially
- introducing a segfault. Mark such REPRESENTATIVES as
- grp_not_necessarilly_dereferenced.
-
- The dereferenced maximum "distance," i.e. the offset + size of the accessed
- part is calculated rather than simple booleans are calculated for each
- pointer parameter to handle cases when only a fraction of the whole
- aggregate is allocated (see testsuite/gcc.c-torture/execute/ipa-sra-2.c for
- an example).
-
- The maximum dereference distances for each pointer parameter and BB are
- already stored in bb_dereference. This routine simply propagates these
- values upwards by propagate_dereference_distances and then compares the
- distances of individual parameters in the ENTRY BB to the equivalent
- distances of each representative of a (fraction of a) parameter. */
-
-static void
-analyze_caller_dereference_legality (vec<access_p> representatives)
-{
- int i;
-
- if (dump_file && (dump_flags & TDF_DETAILS))
- dump_dereferences_table (dump_file,
- "Dereference table before propagation:\n",
- bb_dereferences);
-
- propagate_dereference_distances ();
-
- if (dump_file && (dump_flags & TDF_DETAILS))
- dump_dereferences_table (dump_file,
- "Dereference table after propagation:\n",
- bb_dereferences);
-
- for (i = 0; i < func_param_count; i++)
- {
- struct access *repr = representatives[i];
- int idx = ENTRY_BLOCK_PTR_FOR_FN (cfun)->index * func_param_count + i;
-
- if (!repr || no_accesses_p (repr))
- continue;
-
- do
- {
- if ((repr->offset + repr->size) > bb_dereferences[idx])
- repr->grp_not_necessarilly_dereferenced = 1;
- repr = repr->next_grp;
- }
- while (repr);
- }
-}
-
-/* Return the representative access for the parameter declaration PARM if it is
- a scalar passed by reference which is not written to and the pointer value
- is not used directly. Thus, if it is legal to dereference it in the caller
- and we can rule out modifications through aliases, such parameter should be
- turned into one passed by value. Return NULL otherwise. */
-
-static struct access *
-unmodified_by_ref_scalar_representative (tree parm)
-{
- int i, access_count;
- struct access *repr;
- vec<access_p> *access_vec;
-
- access_vec = get_base_access_vector (parm);
- gcc_assert (access_vec);
- repr = (*access_vec)[0];
- if (repr->write)
- return NULL;
- repr->group_representative = repr;
-
- access_count = access_vec->length ();
- for (i = 1; i < access_count; i++)
- {
- struct access *access = (*access_vec)[i];
- if (access->write)
- return NULL;
- access->group_representative = repr;
- access->next_sibling = repr->next_sibling;
- repr->next_sibling = access;
- }
-
- repr->grp_read = 1;
- repr->grp_scalar_ptr = 1;
- return repr;
-}
-
-/* Return true iff this ACCESS precludes IPA-SRA of the parameter it is
- associated with. REQ_ALIGN is the minimum required alignment. */
-
-static bool
-access_precludes_ipa_sra_p (struct access *access, unsigned int req_align)
-{
- unsigned int exp_align;
- /* Avoid issues such as the second simple testcase in PR 42025. The problem
- is incompatible assign in a call statement (and possibly even in asm
- statements). This can be relaxed by using a new temporary but only for
- non-TREE_ADDRESSABLE types and is probably not worth the complexity. (In
- intraprocedural SRA we deal with this by keeping the old aggregate around,
- something we cannot do in IPA-SRA.) */
- if (access->write
- && (is_gimple_call (access->stmt)
- || gimple_code (access->stmt) == GIMPLE_ASM))
- return true;
-
- exp_align = get_object_alignment (access->expr);
- if (exp_align < req_align)
- return true;
-
- return false;
-}
-
-
-/* Sort collected accesses for parameter PARM, identify representatives for
- each accessed region and link them together. Return NULL if there are
- different but overlapping accesses, return the special ptr value meaning
- there are no accesses for this parameter if that is the case and return the
- first representative otherwise. Set *RO_GRP if there is a group of accesses
- with only read (i.e. no write) accesses. */
-
-static struct access *
-splice_param_accesses (tree parm, bool *ro_grp)
-{
- int i, j, access_count, group_count;
- int total_size = 0;
- struct access *access, *res, **prev_acc_ptr = &res;
- vec<access_p> *access_vec;
-
- access_vec = get_base_access_vector (parm);
- if (!access_vec)
- return &no_accesses_representant;
- access_count = access_vec->length ();
-
- access_vec->qsort (compare_access_positions);
-
- i = 0;
- total_size = 0;
- group_count = 0;
- while (i < access_count)
- {
- bool modification;
- tree a1_alias_type;
- access = (*access_vec)[i];
- modification = access->write;
- if (access_precludes_ipa_sra_p (access, TYPE_ALIGN (access->type)))
- return NULL;
- a1_alias_type = reference_alias_ptr_type (access->expr);
-
- /* Access is about to become group representative unless we find some
- nasty overlap which would preclude us from breaking this parameter
- apart. */
-
- j = i + 1;
- while (j < access_count)
- {
- struct access *ac2 = (*access_vec)[j];
- if (ac2->offset != access->offset)
- {
- /* All or nothing law for parameters. */
- if (access->offset + access->size > ac2->offset)
- return NULL;
- else
- break;
- }
- else if (ac2->size != access->size)
- return NULL;
-
- if (access_precludes_ipa_sra_p (ac2, TYPE_ALIGN (access->type))
- || (ac2->type != access->type
- && (TREE_ADDRESSABLE (ac2->type)
- || TREE_ADDRESSABLE (access->type)))
- || (reference_alias_ptr_type (ac2->expr) != a1_alias_type))
- return NULL;
-
- modification |= ac2->write;
- ac2->group_representative = access;
- ac2->next_sibling = access->next_sibling;
- access->next_sibling = ac2;
- j++;
- }
-
- group_count++;
- access->grp_maybe_modified = modification;
- if (!modification)
- *ro_grp = true;
- *prev_acc_ptr = access;
- prev_acc_ptr = &access->next_grp;
- total_size += access->size;
- i = j;
- }
-
- gcc_assert (group_count > 0);
- return res;
-}
-
-/* Decide whether parameters with representative accesses given by REPR should
- be reduced into components. */
-
-static int
-decide_one_param_reduction (struct access *repr)
-{
- HOST_WIDE_INT total_size, cur_parm_size;
- bool by_ref;
- tree parm;
-
- parm = repr->base;
- cur_parm_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
- gcc_assert (cur_parm_size > 0);
-
- if (POINTER_TYPE_P (TREE_TYPE (parm)))
- by_ref = true;
- else
- by_ref = false;
-
- if (dump_file)
- {
- struct access *acc;
- fprintf (dump_file, "Evaluating PARAM group sizes for ");
- print_generic_expr (dump_file, parm);
- fprintf (dump_file, " (UID: %u): \n", DECL_UID (parm));
- for (acc = repr; acc; acc = acc->next_grp)
- dump_access (dump_file, acc, true);
- }
-
- total_size = 0;
- int new_param_count = 0;
-
- for (; repr; repr = repr->next_grp)
- {
- gcc_assert (parm == repr->base);
-
- /* Taking the address of a non-addressable field is verboten. */
- if (by_ref && repr->non_addressable)
- return 0;
-
- /* Do not decompose a non-BLKmode param in a way that would
- create BLKmode params. Especially for by-reference passing
- (thus, pointer-type param) this is hardly worthwhile. */
- if (DECL_MODE (parm) != BLKmode
- && TYPE_MODE (repr->type) == BLKmode)
- return 0;
-
- if (!by_ref || (!repr->grp_maybe_modified
- && !repr->grp_not_necessarilly_dereferenced))
- total_size += repr->size;
- else
- total_size += cur_parm_size;
-
- new_param_count++;
- }
-
- gcc_assert (new_param_count > 0);
-
- if (!by_ref)
- {
- if (total_size >= cur_parm_size)
- return 0;
- }
- else
- {
- int parm_num_limit;
- if (optimize_function_for_size_p (cfun))
- parm_num_limit = 1;
- else
- parm_num_limit = PARAM_VALUE (PARAM_IPA_SRA_PTR_GROWTH_FACTOR);
-
- if (new_param_count > parm_num_limit
- || total_size > (parm_num_limit * cur_parm_size))
- return 0;
- }
-
- if (dump_file)
- fprintf (dump_file, " ....will be split into %i components\n",
- new_param_count);
- return new_param_count;
-}
-
-/* The order of the following enums is important, we need to do extra work for
- UNUSED_PARAMS, BY_VAL_ACCESSES and UNMODIF_BY_REF_ACCESSES. */
-enum ipa_splicing_result { NO_GOOD_ACCESS, UNUSED_PARAMS, BY_VAL_ACCESSES,
- MODIF_BY_REF_ACCESSES, UNMODIF_BY_REF_ACCESSES };
-
-/* Identify representatives of all accesses to all candidate parameters for
- IPA-SRA. Return result based on what representatives have been found. */
-
-static enum ipa_splicing_result
-splice_all_param_accesses (vec<access_p> &representatives)
-{
- enum ipa_splicing_result result = NO_GOOD_ACCESS;
- tree parm;
- struct access *repr;
-
- representatives.create (func_param_count);
-
- for (parm = DECL_ARGUMENTS (current_function_decl);
- parm;
- parm = DECL_CHAIN (parm))
- {
- if (is_unused_scalar_param (parm))
- {
- representatives.quick_push (&no_accesses_representant);
- if (result == NO_GOOD_ACCESS)
- result = UNUSED_PARAMS;
- }
- else if (POINTER_TYPE_P (TREE_TYPE (parm))
- && is_gimple_reg_type (TREE_TYPE (TREE_TYPE (parm)))
- && bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
- {
- repr = unmodified_by_ref_scalar_representative (parm);
- representatives.quick_push (repr);
- if (repr)
- result = UNMODIF_BY_REF_ACCESSES;
- }
- else if (bitmap_bit_p (candidate_bitmap, DECL_UID (parm)))
- {
- bool ro_grp = false;
- repr = splice_param_accesses (parm, &ro_grp);
- representatives.quick_push (repr);
-
- if (repr && !no_accesses_p (repr))
- {
- if (POINTER_TYPE_P (TREE_TYPE (parm)))
- {
- if (ro_grp)
- result = UNMODIF_BY_REF_ACCESSES;
- else if (result < MODIF_BY_REF_ACCESSES)
- result = MODIF_BY_REF_ACCESSES;
- }
- else if (result < BY_VAL_ACCESSES)
- result = BY_VAL_ACCESSES;
- }
- else if (no_accesses_p (repr) && (result == NO_GOOD_ACCESS))
- result = UNUSED_PARAMS;
- }
- else
- representatives.quick_push (NULL);
- }
-
- if (result == NO_GOOD_ACCESS)
- {
- representatives.release ();
- return NO_GOOD_ACCESS;
- }
-
- return result;
-}
-
-/* Return the index of BASE in PARMS. Abort if it is not found. */
-
-static inline int
-get_param_index (tree base, vec<tree> parms)
-{
- int i, len;
-
- len = parms.length ();
- for (i = 0; i < len; i++)
- if (parms[i] == base)
- return i;
- gcc_unreachable ();
-}
-
-/* Convert the decisions made at the representative level into compact
- parameter adjustments. REPRESENTATIVES are pointers to first
- representatives of each param accesses, ADJUSTMENTS_COUNT is the expected
- final number of adjustments. */
-
-static ipa_parm_adjustment_vec
-turn_representatives_into_adjustments (vec<access_p> representatives,
- int adjustments_count)
-{
- vec<tree> parms;
- ipa_parm_adjustment_vec adjustments;
- tree parm;
- int i;
-
- gcc_assert (adjustments_count > 0);
- parms = ipa_get_vector_of_formal_parms (current_function_decl);
- adjustments.create (adjustments_count);
- parm = DECL_ARGUMENTS (current_function_decl);
- for (i = 0; i < func_param_count; i++, parm = DECL_CHAIN (parm))
- {
- struct access *repr = representatives[i];
-
- if (!repr || no_accesses_p (repr))
- {
- struct ipa_parm_adjustment adj;
-
- memset (&adj, 0, sizeof (adj));
- adj.base_index = get_param_index (parm, parms);
- adj.base = parm;
- if (!repr)
- adj.op = IPA_PARM_OP_COPY;
- else
- adj.op = IPA_PARM_OP_REMOVE;
- adj.arg_prefix = "ISRA";
- adjustments.quick_push (adj);
- }
- else
- {
- struct ipa_parm_adjustment adj;
- int index = get_param_index (parm, parms);
-
- for (; repr; repr = repr->next_grp)
- {
- memset (&adj, 0, sizeof (adj));
- gcc_assert (repr->base == parm);
- adj.base_index = index;
- adj.base = repr->base;
- adj.type = repr->type;
- adj.alias_ptr_type = reference_alias_ptr_type (repr->expr);
- adj.offset = repr->offset;
- adj.reverse = repr->reverse;
- adj.by_ref = (POINTER_TYPE_P (TREE_TYPE (repr->base))
- && (repr->grp_maybe_modified
- || repr->grp_not_necessarilly_dereferenced));
- adj.arg_prefix = "ISRA";
- adjustments.quick_push (adj);
- }
- }
- }
- parms.release ();
- return adjustments;
-}
-
-/* Analyze the collected accesses and produce a plan what to do with the
- parameters in the form of adjustments, NULL meaning nothing. */
-
-static ipa_parm_adjustment_vec
-analyze_all_param_acesses (void)
-{
- enum ipa_splicing_result repr_state;
- bool proceed = false;
- int i, adjustments_count = 0;
- vec<access_p> representatives;
- ipa_parm_adjustment_vec adjustments;
-
- repr_state = splice_all_param_accesses (representatives);
- if (repr_state == NO_GOOD_ACCESS)
- return ipa_parm_adjustment_vec ();
-
- /* If there are any parameters passed by reference which are not modified
- directly, we need to check whether they can be modified indirectly. */
- if (repr_state == UNMODIF_BY_REF_ACCESSES)
- {
- analyze_caller_dereference_legality (representatives);
- analyze_modified_params (representatives);
- }
-
- for (i = 0; i < func_param_count; i++)
- {
- struct access *repr = representatives[i];
-
- if (repr && !no_accesses_p (repr))
- {
- if (repr->grp_scalar_ptr)
- {
- adjustments_count++;
- if (repr->grp_not_necessarilly_dereferenced
- || repr->grp_maybe_modified)
- representatives[i] = NULL;
- else
- {
- proceed = true;
- sra_stats.scalar_by_ref_to_by_val++;
- }
- }
- else
- {
- int new_components = decide_one_param_reduction (repr);
-
- if (new_components == 0)
- {
- representatives[i] = NULL;
- adjustments_count++;
- }
- else
- {
- adjustments_count += new_components;
- sra_stats.aggregate_params_reduced++;
- sra_stats.param_reductions_created += new_components;
- proceed = true;
- }
- }
- }
- else
- {
- if (no_accesses_p (repr))
- {
- proceed = true;
- sra_stats.deleted_unused_parameters++;
- }
- adjustments_count++;
- }
- }
-
- if (!proceed && dump_file)
- fprintf (dump_file, "NOT proceeding to change params.\n");
-
- if (proceed)
- adjustments = turn_representatives_into_adjustments (representatives,
- adjustments_count);
- else
- adjustments = ipa_parm_adjustment_vec ();
-
- representatives.release ();
- return adjustments;
-}
-
-/* If a parameter replacement identified by ADJ does not yet exist in the form
- of declaration, create it and record it, otherwise return the previously
- created one. */
-
-static tree
-get_replaced_param_substitute (struct ipa_parm_adjustment *adj)
-{
- tree repl;
- if (!adj->new_ssa_base)
- {
- char *pretty_name = make_fancy_name (adj->base);
-
- repl = create_tmp_reg (TREE_TYPE (adj->base), "ISR");
- DECL_NAME (repl) = get_identifier (pretty_name);
- DECL_NAMELESS (repl) = 1;
- obstack_free (&name_obstack, pretty_name);
-
- adj->new_ssa_base = repl;
- }
- else
- repl = adj->new_ssa_base;
- return repl;
-}
-
-/* Find the first adjustment for a particular parameter BASE in a vector of
- ADJUSTMENTS which is not a copy_param. Return NULL if there is no such
- adjustment. */
-
-static struct ipa_parm_adjustment *
-get_adjustment_for_base (ipa_parm_adjustment_vec adjustments, tree base)
-{
- int i, len;
-
- len = adjustments.length ();
- for (i = 0; i < len; i++)
- {
- struct ipa_parm_adjustment *adj;
-
- adj = &adjustments[i];
- if (adj->op != IPA_PARM_OP_COPY && adj->base == base)
- return adj;
- }
-
- return NULL;
-}
-
-/* If OLD_NAME, which is being defined by statement STMT, is an SSA_NAME of a
- parameter which is to be removed because its value is not used, create a new
- SSA_NAME relating to a replacement VAR_DECL, replace all uses of the
- original with it and return it. If there is no need to re-map, return NULL.
- ADJUSTMENTS is a pointer to a vector of IPA-SRA adjustments. */
-
-static tree
-replace_removed_params_ssa_names (tree old_name, gimple *stmt,
- ipa_parm_adjustment_vec adjustments)
-{
- struct ipa_parm_adjustment *adj;
- tree decl, repl, new_name;
-
- if (TREE_CODE (old_name) != SSA_NAME)
- return NULL;
-
- decl = SSA_NAME_VAR (old_name);
- if (decl == NULL_TREE
- || TREE_CODE (decl) != PARM_DECL)
- return NULL;
-
- adj = get_adjustment_for_base (adjustments, decl);
- if (!adj)
- return NULL;
-
- repl = get_replaced_param_substitute (adj);
- new_name = make_ssa_name (repl, stmt);
- SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_name)
- = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (old_name);
-
- if (dump_file)
- {
- fprintf (dump_file, "replacing an SSA name of a removed param ");
- print_generic_expr (dump_file, old_name);
- fprintf (dump_file, " with ");
- print_generic_expr (dump_file, new_name);
- fprintf (dump_file, "\n");
- }
-
- replace_uses_by (old_name, new_name);
- return new_name;
-}
-
-/* If the statement STMT contains any expressions that need to replaced with a
- different one as noted by ADJUSTMENTS, do so. Handle any potential type
- incompatibilities (GSI is used to accommodate conversion statements and must
- point to the statement). Return true iff the statement was modified. */
-
-static bool
-sra_ipa_modify_assign (gimple *stmt, gimple_stmt_iterator *gsi,
- ipa_parm_adjustment_vec adjustments)
-{
- tree *lhs_p, *rhs_p;
- bool any;
-
- if (!gimple_assign_single_p (stmt))
- return false;
-
- rhs_p = gimple_assign_rhs1_ptr (stmt);
- lhs_p = gimple_assign_lhs_ptr (stmt);
-
- any = ipa_modify_expr (rhs_p, false, adjustments);
- any |= ipa_modify_expr (lhs_p, false, adjustments);
- if (any)
- {
- tree new_rhs = NULL_TREE;
-
- if (!useless_type_conversion_p (TREE_TYPE (*lhs_p), TREE_TYPE (*rhs_p)))
- {
- if (TREE_CODE (*rhs_p) == CONSTRUCTOR)
- {
- /* V_C_Es of constructors can cause trouble (PR 42714). */
- if (is_gimple_reg_type (TREE_TYPE (*lhs_p)))
- *rhs_p = build_zero_cst (TREE_TYPE (*lhs_p));
- else
- *rhs_p = build_constructor (TREE_TYPE (*lhs_p),
- NULL);
- }
- else
- new_rhs = fold_build1_loc (gimple_location (stmt),
- VIEW_CONVERT_EXPR, TREE_TYPE (*lhs_p),
- *rhs_p);
- }
- else if (REFERENCE_CLASS_P (*rhs_p)
- && is_gimple_reg_type (TREE_TYPE (*lhs_p))
- && !is_gimple_reg (*lhs_p))
- /* This can happen when an assignment in between two single field
- structures is turned into an assignment in between two pointers to
- scalars (PR 42237). */
- new_rhs = *rhs_p;
-
- if (new_rhs)
- {
- tree tmp = force_gimple_operand_gsi (gsi, new_rhs, true, NULL_TREE,
- true, GSI_SAME_STMT);
-
- gimple_assign_set_rhs_from_tree (gsi, tmp);
- }
-
- return true;
- }
-
- return false;
-}
-
-/* Traverse the function body and all modifications as described in
- ADJUSTMENTS. Return true iff the CFG has been changed. */
-
-bool
-ipa_sra_modify_function_body (ipa_parm_adjustment_vec adjustments)
-{
- bool cfg_changed = false;
- basic_block bb;
-
- FOR_EACH_BB_FN (bb, cfun)
- {
- gimple_stmt_iterator gsi;
-
- for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
- {
- gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
- tree new_lhs, old_lhs = gimple_phi_result (phi);
- new_lhs = replace_removed_params_ssa_names (old_lhs, phi, adjustments);
- if (new_lhs)
- {
- gimple_phi_set_result (phi, new_lhs);
- release_ssa_name (old_lhs);
- }
- }
-
- gsi = gsi_start_bb (bb);
- while (!gsi_end_p (gsi))
- {
- gimple *stmt = gsi_stmt (gsi);
- bool modified = false;
- tree *t;
- unsigned i;
-
- switch (gimple_code (stmt))
- {
- case GIMPLE_RETURN:
- t = gimple_return_retval_ptr (as_a <greturn *> (stmt));
- if (*t != NULL_TREE)
- modified |= ipa_modify_expr (t, true, adjustments);
- break;
-
- case GIMPLE_ASSIGN:
- modified |= sra_ipa_modify_assign (stmt, &gsi, adjustments);
- break;
-
- case GIMPLE_CALL:
- /* Operands must be processed before the lhs. */
- for (i = 0; i < gimple_call_num_args (stmt); i++)
- {
- t = gimple_call_arg_ptr (stmt, i);
- modified |= ipa_modify_expr (t, true, adjustments);
- }
-
- if (gimple_call_lhs (stmt))
- {
- t = gimple_call_lhs_ptr (stmt);
- modified |= ipa_modify_expr (t, false, adjustments);
- }
- break;
-
- case GIMPLE_ASM:
- {
- gasm *asm_stmt = as_a <gasm *> (stmt);
- for (i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
- {
- t = &TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
- modified |= ipa_modify_expr (t, true, adjustments);
- }
- for (i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
- {
- t = &TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
- modified |= ipa_modify_expr (t, false, adjustments);
- }
- }
- break;
-
- default:
- break;
- }
-
- def_operand_p defp;
- ssa_op_iter iter;
- FOR_EACH_SSA_DEF_OPERAND (defp, stmt, iter, SSA_OP_DEF)
- {
- tree old_def = DEF_FROM_PTR (defp);
- if (tree new_def = replace_removed_params_ssa_names (old_def, stmt,
- adjustments))
- {
- SET_DEF (defp, new_def);
- release_ssa_name (old_def);
- modified = true;
- }
- }
-
- if (modified)
- {
- update_stmt (stmt);
- if (maybe_clean_eh_stmt (stmt)
- && gimple_purge_dead_eh_edges (gimple_bb (stmt)))
- cfg_changed = true;
- }
- gsi_next (&gsi);
- }
- }
-
- return cfg_changed;
-}
-
-/* Call gimple_debug_bind_reset_value on all debug statements describing
- gimple register parameters that are being removed or replaced. */
-
-static void
-sra_ipa_reset_debug_stmts (ipa_parm_adjustment_vec adjustments)
-{
- int i, len;
- gimple_stmt_iterator *gsip = NULL, gsi;
-
- if (MAY_HAVE_DEBUG_STMTS && single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
- {
- gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
- gsip = &gsi;
- }
- len = adjustments.length ();
- for (i = 0; i < len; i++)
- {
- struct ipa_parm_adjustment *adj;
- imm_use_iterator ui;
- gimple *stmt;
- gdebug *def_temp;
- tree name, vexpr, copy = NULL_TREE;
- use_operand_p use_p;
-
- adj = &adjustments[i];
- if (adj->op == IPA_PARM_OP_COPY || !is_gimple_reg (adj->base))
- continue;
- name = ssa_default_def (cfun, adj->base);
- vexpr = NULL;
- if (name)
- FOR_EACH_IMM_USE_STMT (stmt, ui, name)
- {
- if (gimple_clobber_p (stmt))
- {
- gimple_stmt_iterator cgsi = gsi_for_stmt (stmt);
- unlink_stmt_vdef (stmt);
- gsi_remove (&cgsi, true);
- release_defs (stmt);
- continue;
- }
- /* All other users must have been removed by
- ipa_sra_modify_function_body. */
- gcc_assert (is_gimple_debug (stmt));
- if (vexpr == NULL && gsip != NULL)
- {
- gcc_assert (TREE_CODE (adj->base) == PARM_DECL);
- vexpr = make_node (DEBUG_EXPR_DECL);
- def_temp = gimple_build_debug_source_bind (vexpr, adj->base,
- NULL);
- DECL_ARTIFICIAL (vexpr) = 1;
- TREE_TYPE (vexpr) = TREE_TYPE (name);
- SET_DECL_MODE (vexpr, DECL_MODE (adj->base));
- gsi_insert_before (gsip, def_temp, GSI_SAME_STMT);
- }
- if (vexpr)
- {
- FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
- SET_USE (use_p, vexpr);
- }
- else
- gimple_debug_bind_reset_value (stmt);
- update_stmt (stmt);
- }
- /* Create a VAR_DECL for debug info purposes. */
- if (!DECL_IGNORED_P (adj->base))
- {
- copy = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
- VAR_DECL, DECL_NAME (adj->base),
- TREE_TYPE (adj->base));
- if (DECL_PT_UID_SET_P (adj->base))
- SET_DECL_PT_UID (copy, DECL_PT_UID (adj->base));
- TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (adj->base);
- TREE_READONLY (copy) = TREE_READONLY (adj->base);
- TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (adj->base);
- DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (adj->base);
- DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (adj->base);
- DECL_IGNORED_P (copy) = DECL_IGNORED_P (adj->base);
- DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (adj->base);
- DECL_SEEN_IN_BIND_EXPR_P (copy) = 1;
- SET_DECL_RTL (copy, 0);
- TREE_USED (copy) = 1;
- DECL_CONTEXT (copy) = current_function_decl;
- add_local_decl (cfun, copy);
- DECL_CHAIN (copy) =
- BLOCK_VARS (DECL_INITIAL (current_function_decl));
- BLOCK_VARS (DECL_INITIAL (current_function_decl)) = copy;
- }
- if (gsip != NULL && copy && target_for_debug_bind (adj->base))
- {
- gcc_assert (TREE_CODE (adj->base) == PARM_DECL);
- if (vexpr)
- def_temp = gimple_build_debug_bind (copy, vexpr, NULL);
- else
- def_temp = gimple_build_debug_source_bind (copy, adj->base,
- NULL);
- gsi_insert_before (gsip, def_temp, GSI_SAME_STMT);
- }
- }
-}
-
-/* Return false if all callers have at least as many actual arguments as there
- are formal parameters in the current function and that their types
- match. */
-
-static bool
-some_callers_have_mismatched_arguments_p (struct cgraph_node *node,
- void *data ATTRIBUTE_UNUSED)
-{
- struct cgraph_edge *cs;
- for (cs = node->callers; cs; cs = cs->next_caller)
- if (!cs->call_stmt || !callsite_arguments_match_p (cs->call_stmt))
- return true;
-
- return false;
-}
-
-/* Return false if all callers have vuse attached to a call statement. */
-
-static bool
-some_callers_have_no_vuse_p (struct cgraph_node *node,
- void *data ATTRIBUTE_UNUSED)
-{
- struct cgraph_edge *cs;
- for (cs = node->callers; cs; cs = cs->next_caller)
- if (!cs->call_stmt || !gimple_vuse (cs->call_stmt))
- return true;
-
- return false;
-}
-
-/* Convert all callers of NODE. */
-
-static bool
-convert_callers_for_node (struct cgraph_node *node,
- void *data)
-{
- ipa_parm_adjustment_vec *adjustments = (ipa_parm_adjustment_vec *) data;
- bitmap recomputed_callers = BITMAP_ALLOC (NULL);
- struct cgraph_edge *cs;
-
- for (cs = node->callers; cs; cs = cs->next_caller)
- {
- push_cfun (DECL_STRUCT_FUNCTION (cs->caller->decl));
-
- if (dump_file)
- fprintf (dump_file, "Adjusting call %s -> %s\n",
- cs->caller->dump_name (), cs->callee->dump_name ());
-
- ipa_modify_call_arguments (cs, cs->call_stmt, *adjustments);
-
- pop_cfun ();
- }
-
- for (cs = node->callers; cs; cs = cs->next_caller)
- if (bitmap_set_bit (recomputed_callers, cs->caller->uid)
- && gimple_in_ssa_p (DECL_STRUCT_FUNCTION (cs->caller->decl)))
- compute_fn_summary (cs->caller, true);
- BITMAP_FREE (recomputed_callers);
-
- return true;
-}
-
-/* Convert all callers of NODE to pass parameters as given in ADJUSTMENTS. */
-
-static void
-convert_callers (struct cgraph_node *node, tree old_decl,
- ipa_parm_adjustment_vec adjustments)
-{
- basic_block this_block;
-
- node->call_for_symbol_and_aliases (convert_callers_for_node,
- &adjustments, false);
-
- if (!encountered_recursive_call)
- return;
-
- FOR_EACH_BB_FN (this_block, cfun)
- {
- gimple_stmt_iterator gsi;
-
- for (gsi = gsi_start_bb (this_block); !gsi_end_p (gsi); gsi_next (&gsi))
- {
- gcall *stmt;
- tree call_fndecl;
- stmt = dyn_cast <gcall *> (gsi_stmt (gsi));
- if (!stmt)
- continue;
- call_fndecl = gimple_call_fndecl (stmt);
- if (call_fndecl == old_decl)
- {
- if (dump_file)
- fprintf (dump_file, "Adjusting recursive call");
- gimple_call_set_fndecl (stmt, node->decl);
- ipa_modify_call_arguments (NULL, stmt, adjustments);
- }
- }
- }
-
- return;
-}
-
-/* Perform all the modification required in IPA-SRA for NODE to have parameters
- as given in ADJUSTMENTS. Return true iff the CFG has been changed. */
-
-static bool
-modify_function (struct cgraph_node *node, ipa_parm_adjustment_vec adjustments)
-{
- struct cgraph_node *new_node;
- bool cfg_changed;
-
- cgraph_edge::rebuild_edges ();
- free_dominance_info (CDI_DOMINATORS);
- pop_cfun ();
-
- /* This must be done after rebuilding cgraph edges for node above.
- Otherwise any recursive calls to node that are recorded in
- redirect_callers will be corrupted. */
- vec<cgraph_edge *> redirect_callers = node->collect_callers ();
- new_node = node->create_version_clone_with_body (redirect_callers, NULL,
- NULL, false, NULL, NULL,
- "isra");
- redirect_callers.release ();
-
- push_cfun (DECL_STRUCT_FUNCTION (new_node->decl));
- ipa_modify_formal_parameters (current_function_decl, adjustments);
- cfg_changed = ipa_sra_modify_function_body (adjustments);
- sra_ipa_reset_debug_stmts (adjustments);
- convert_callers (new_node, node->decl, adjustments);
- new_node->make_local ();
- return cfg_changed;
-}
-
-/* Means of communication between ipa_sra_check_caller and
- ipa_sra_preliminary_function_checks. */
-
-struct ipa_sra_check_caller_data
-{
- bool has_callers;
- bool bad_arg_alignment;
- bool has_thunk;
-};
-
-/* If NODE has a caller, mark that fact in DATA which is pointer to
- ipa_sra_check_caller_data. Also check all aggregate arguments in all known
- calls if they are unit aligned and if not, set the appropriate flag in DATA
- too. */
-
-static bool
-ipa_sra_check_caller (struct cgraph_node *node, void *data)
-{
- if (!node->callers)
- return false;
-
- struct ipa_sra_check_caller_data *iscc;
- iscc = (struct ipa_sra_check_caller_data *) data;
- iscc->has_callers = true;
-
- for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
- {
- if (cs->caller->thunk.thunk_p)
- {
- iscc->has_thunk = true;
- return true;
- }
- gimple *call_stmt = cs->call_stmt;
- unsigned count = gimple_call_num_args (call_stmt);
- for (unsigned i = 0; i < count; i++)
- {
- tree arg = gimple_call_arg (call_stmt, i);
- if (is_gimple_reg (arg))
- continue;
-
- tree offset;
- poly_int64 bitsize, bitpos;
- machine_mode mode;
- int unsignedp, reversep, volatilep = 0;
- get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
- &unsignedp, &reversep, &volatilep);
- if (!multiple_p (bitpos, BITS_PER_UNIT))
- {
- iscc->bad_arg_alignment = true;
- return true;
- }
- }
- }
-
- return false;
-}
-
-/* Return false the function is apparently unsuitable for IPA-SRA based on it's
- attributes, return true otherwise. NODE is the cgraph node of the current
- function. */
-
-static bool
-ipa_sra_preliminary_function_checks (struct cgraph_node *node)
-{
- if (!node->can_be_local_p ())
- {
- if (dump_file)
- fprintf (dump_file, "Function not local to this compilation unit.\n");
- return false;
- }
-
- if (!node->local.can_change_signature)
- {
- if (dump_file)
- fprintf (dump_file, "Function can not change signature.\n");
- return false;
- }
-
- if (!tree_versionable_function_p (node->decl))
- {
- if (dump_file)
- fprintf (dump_file, "Function is not versionable.\n");
- return false;
- }
-
- if (!opt_for_fn (node->decl, optimize)
- || !opt_for_fn (node->decl, flag_ipa_sra))
- {
- if (dump_file)
- fprintf (dump_file, "Function not optimized.\n");
- return false;
- }
-
- if (DECL_VIRTUAL_P (current_function_decl))
- {
- if (dump_file)
- fprintf (dump_file, "Function is a virtual method.\n");
- return false;
- }
-
- if ((DECL_ONE_ONLY (node->decl) || DECL_EXTERNAL (node->decl))
- && ipa_fn_summaries->get_create (node)->size >= MAX_INLINE_INSNS_AUTO)
- {
- if (dump_file)
- fprintf (dump_file, "Function too big to be made truly local.\n");
- return false;
- }
-
- if (cfun->stdarg)
- {
- if (dump_file)
- fprintf (dump_file, "Function uses stdarg. \n");
- return false;
- }
-
- if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl)))
- return false;
-
- if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
- {
- if (dump_file)
- fprintf (dump_file, "Always inline function will be inlined "
- "anyway. \n");
- return false;
- }
-
- struct ipa_sra_check_caller_data iscc;
- memset (&iscc, 0, sizeof(iscc));
- node->call_for_symbol_and_aliases (ipa_sra_check_caller, &iscc, true);
- if (!iscc.has_callers)
- {
- if (dump_file)
- fprintf (dump_file,
- "Function has no callers in this compilation unit.\n");
- return false;
- }
-
- if (iscc.bad_arg_alignment)
- {
- if (dump_file)
- fprintf (dump_file,
- "A function call has an argument with non-unit alignment.\n");
- return false;
- }
-
- if (iscc.has_thunk)
- {
- if (dump_file)
- fprintf (dump_file,
- "A has thunk.\n");
- return false;
- }
-
- return true;
-}
-
-/* Perform early interprocedural SRA. */
-
-static unsigned int
-ipa_early_sra (void)
-{
- struct cgraph_node *node = cgraph_node::get (current_function_decl);
- ipa_parm_adjustment_vec adjustments;
- int ret = 0;
-
- if (!ipa_sra_preliminary_function_checks (node))
- return 0;
-
- sra_initialize ();
- sra_mode = SRA_MODE_EARLY_IPA;
-
- if (!find_param_candidates ())
- {
- if (dump_file)
- fprintf (dump_file, "Function has no IPA-SRA candidates.\n");
- goto simple_out;
- }
-
- if (node->call_for_symbol_and_aliases
- (some_callers_have_mismatched_arguments_p, NULL, true))
- {
- if (dump_file)
- fprintf (dump_file, "There are callers with insufficient number of "
- "arguments or arguments with type mismatches.\n");
- goto simple_out;
- }
-
- if (node->call_for_symbol_and_aliases
- (some_callers_have_no_vuse_p, NULL, true))
- {
- if (dump_file)
- fprintf (dump_file, "There are callers with no VUSE attached "
- "to a call stmt.\n");
- goto simple_out;
- }
-
- bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
- func_param_count
- * last_basic_block_for_fn (cfun));
- final_bbs = BITMAP_ALLOC (NULL);
-
- scan_function ();
- if (encountered_apply_args)
- {
- if (dump_file)
- fprintf (dump_file, "Function calls __builtin_apply_args().\n");
- goto out;
- }
-
- if (encountered_unchangable_recursive_call)
- {
- if (dump_file)
- fprintf (dump_file, "Function calls itself with insufficient "
- "number of arguments.\n");
- goto out;
- }
-
- adjustments = analyze_all_param_acesses ();
- if (!adjustments.exists ())
- goto out;
- if (dump_file)
- ipa_dump_param_adjustments (dump_file, adjustments, current_function_decl);
-
- if (modify_function (node, adjustments))
- ret = TODO_update_ssa | TODO_cleanup_cfg;
- else
- ret = TODO_update_ssa;
- adjustments.release ();
-
- statistics_counter_event (cfun, "Unused parameters deleted",
- sra_stats.deleted_unused_parameters);
- statistics_counter_event (cfun, "Scalar parameters converted to by-value",
- sra_stats.scalar_by_ref_to_by_val);
- statistics_counter_event (cfun, "Aggregate parameters broken up",
- sra_stats.aggregate_params_reduced);
- statistics_counter_event (cfun, "Aggregate parameter components created",
- sra_stats.param_reductions_created);
-
- out:
- BITMAP_FREE (final_bbs);
- free (bb_dereferences);
- simple_out:
- sra_deinitialize ();
- return ret;
-}
-
-namespace {
-
-const pass_data pass_data_early_ipa_sra =
-{
- GIMPLE_PASS, /* type */
- "eipa_sra", /* name */
- OPTGROUP_NONE, /* optinfo_flags */
- TV_IPA_SRA, /* tv_id */
- 0, /* properties_required */
- 0, /* properties_provided */
- 0, /* properties_destroyed */
- 0, /* todo_flags_start */
- TODO_dump_symtab, /* todo_flags_finish */
-};
-
-class pass_early_ipa_sra : public gimple_opt_pass
-{
-public:
- pass_early_ipa_sra (gcc::context *ctxt)
- : gimple_opt_pass (pass_data_early_ipa_sra, ctxt)
- {}
-
- /* opt_pass methods: */
- virtual bool gate (function *) { return flag_ipa_sra && dbg_cnt (eipa_sra); }
- virtual unsigned int execute (function *) { return ipa_early_sra (); }
-
-}; // class pass_early_ipa_sra
-
-} // anon namespace
-
-gimple_opt_pass *
-make_pass_early_ipa_sra (gcc::context *ctxt)
-{
- return new pass_early_ipa_sra (ctxt);
-}
projects / thirdparty / gcc.git / blobdiff