/* "Bag-of-pages" garbage collector for the GNU compiler.
- Copyright (C) 1999-2013 Free Software Foundation, Inc.
+ Copyright (C) 1999-2019 Free Software Foundation, Inc.
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "tm.h"
+#include "backend.h"
+#include "alias.h"
#include "tree.h"
#include "rtl.h"
+#include "memmodel.h"
#include "tm_p.h"
#include "diagnostic-core.h"
#include "flags.h"
-#include "ggc.h"
#include "ggc-internal.h"
#include "timevar.h"
#include "params.h"
-#include "tree-ssa.h"
+#include "cgraph.h"
#include "cfgloop.h"
#include "plugin.h"
3: Object allocations as well.
4: Object marks as well. */
#define GGC_DEBUG_LEVEL (0)
-\f
-#ifndef HOST_BITS_PER_PTR
-#define HOST_BITS_PER_PTR HOST_BITS_PER_LONG
-#endif
-
\f
/* A two-level tree is used to look up the page-entry for a given
pointer. Two chunks of the pointer's bits are extracted to index
struct max_alignment {
char c;
union {
- HOST_WIDEST_INT i;
+ int64_t i;
void *p;
} u;
};
sizeof (struct function),
sizeof (struct basic_block_def),
sizeof (struct cgraph_node),
- sizeof (struct loop),
+ sizeof (class loop),
};
/* The total number of orders. */
#define ROUND_UP_VALUE(x, f) ((f) - 1 - ((f) - 1 + (x)) % (f))
-/* Compute the smallest multiple of F that is >= X. */
-
-#define ROUND_UP(x, f) (CEIL (x, f) * (f))
-
/* Round X to next multiple of the page size */
-#define PAGE_ALIGN(x) (((x) + G.pagesize - 1) & ~(G.pagesize - 1))
+#define PAGE_ALIGN(x) ROUND_UP ((x), G.pagesize)
/* The Ith entry is the number of objects on a page or order I. */
/* A page_entry records the status of an allocation page. This
structure is dynamically sized to fit the bitmap in_use_p. */
-typedef struct page_entry
+struct page_entry
{
/* The next page-entry with objects of the same size, or NULL if
this is the last page-entry. */
Nth bit is one if the Nth object on this page is allocated. This
array is dynamically sized. */
unsigned long in_use_p[1];
-} page_entry;
+};
#ifdef USING_MALLOC_PAGE_GROUPS
/* A page_group describes a large allocation from malloc, from which
we parcel out aligned pages. */
-typedef struct page_group
+struct page_group
{
/* A linked list of all extant page groups. */
struct page_group *next;
/* A bitmask of pages in use. */
unsigned int in_use;
-} page_group;
+};
#endif
#if HOST_BITS_PER_PTR <= 32
#endif
+class finalizer
+{
+public:
+ finalizer (void *addr, void (*f)(void *)) : m_addr (addr), m_function (f) {}
+
+ void *addr () const { return m_addr; }
+
+ void call () const { m_function (m_addr); }
+
+private:
+ void *m_addr;
+ void (*m_function)(void *);
+};
+
+class vec_finalizer
+{
+public:
+ vec_finalizer (uintptr_t addr, void (*f)(void *), size_t s, size_t n) :
+ m_addr (addr), m_function (f), m_object_size (s), m_n_objects (n) {}
+
+ void call () const
+ {
+ for (size_t i = 0; i < m_n_objects; i++)
+ m_function (reinterpret_cast<void *> (m_addr + (i * m_object_size)));
+ }
+
+ void *addr () const { return reinterpret_cast<void *> (m_addr); }
+
+private:
+ uintptr_t m_addr;
+ void (*m_function)(void *);
+ size_t m_object_size;
+ size_t m_n_objects;
+};
+
#ifdef ENABLE_GC_ALWAYS_COLLECT
/* List of free objects to be verified as actually free on the
next collection. */
#endif
/* The rest of the global variables. */
-static struct globals
+static struct ggc_globals
{
/* The Nth element in this array is a page with objects of size 2^N.
If there are any pages with free objects, they will be at the
better runtime data access pattern. */
unsigned long **save_in_use;
+ /* Finalizers for single objects. The first index is collection_depth. */
+ vec<vec<finalizer> > finalizers;
+
+ /* Finalizers for vectors of objects. */
+ vec<vec<vec_finalizer> > vec_finalizers;
+
#ifdef ENABLE_GC_ALWAYS_COLLECT
/* List of free objects to be verified as actually free on the
next collection. */
} stats;
} G;
+/* True if a gc is currently taking place. */
+
+static bool in_gc = false;
+
/* The size in bytes required to maintain a bitmap for the objects
on a page-entry. */
#define BITMAP_SIZE(Num_objects) \
- (CEIL ((Num_objects), HOST_BITS_PER_LONG) * sizeof(long))
+ (CEIL ((Num_objects), HOST_BITS_PER_LONG) * sizeof (long))
/* Allocate pages in chunks of this size, to throttle calls to memory
allocation routines. The first page is used, the rest go onto the
/* Initial guess as to how many page table entries we might need. */
#define INITIAL_PTE_COUNT 128
\f
-static int ggc_allocated_p (const void *);
static page_entry *lookup_page_table_entry (const void *);
static void set_page_table_entry (void *, page_entry *);
#ifdef USING_MMAP
#define save_in_use_p(__p) \
(save_in_use_p_i (__p->index_by_depth))
-/* Returns nonzero if P was allocated in GC'able memory. */
+/* Traverse the page table and find the entry for a page.
+ If the object wasn't allocated in GC return NULL. */
-static inline int
-ggc_allocated_p (const void *p)
+static inline page_entry *
+safe_lookup_page_table_entry (const void *p)
{
page_entry ***base;
size_t L1, L2;
while (1)
{
if (table == NULL)
- return 0;
+ return NULL;
if (table->high_bits == high_bits)
break;
table = table->next;
/* Extract the level 1 and 2 indices. */
L1 = LOOKUP_L1 (p);
L2 = LOOKUP_L2 (p);
+ if (! base[L1])
+ return NULL;
- return base[L1] && base[L1][L2];
+ return base[L1][L2];
}
/* Traverse the page table and find the entry for a page.
page = alloc_anon (NULL, G.pagesize * GGC_QUIRE_SIZE, false);
if (page == NULL)
{
- page = alloc_anon(NULL, G.pagesize, true);
+ page = alloc_anon (NULL, G.pagesize, true);
entries = 1;
}
if (GGC_DEBUG_LEVEL >= 2)
fprintf (G.debug_file,
"Allocating page at %p, object size=%lu, data %p-%p\n",
- (void *) entry, (unsigned long) OBJECT_SIZE (order), page,
- page + entry_size - 1);
+ (void *) entry, (unsigned long) OBJECT_SIZE (order),
+ (void *) page, (void *) (page + entry_size - 1));
return entry;
}
if (GGC_DEBUG_LEVEL >= 2)
fprintf (G.debug_file,
"Deallocating page at %p, data %p-%p\n", (void *) entry,
- entry->page, entry->page + entry->bytes - 1);
+ (void *) entry->page, (void *) (entry->page + entry->bytes - 1));
/* Mark the page as inaccessible. Discard the handle to avoid handle
leak. */
return size;
}
+/* Push a finalizer onto the appropriate vec. */
+
+static void
+add_finalizer (void *result, void (*f)(void *), size_t s, size_t n)
+{
+ if (f == NULL)
+ /* No finalizer. */;
+ else if (n == 1)
+ {
+ finalizer fin (result, f);
+ G.finalizers[G.context_depth].safe_push (fin);
+ }
+ else
+ {
+ vec_finalizer fin (reinterpret_cast<uintptr_t> (result), f, s, n);
+ G.vec_finalizers[G.context_depth].safe_push (fin);
+ }
+}
+
/* Allocate a chunk of memory of SIZE bytes. Its contents are undefined. */
void *
-ggc_internal_alloc_stat (size_t size MEM_STAT_DECL)
+ggc_internal_alloc (size_t size, void (*f)(void *), size_t s, size_t n
+ MEM_STAT_DECL)
{
size_t order, word, bit, object_offset, object_size;
struct page_entry *entry;
/* For timevar statistics. */
timevar_ggc_mem_total += object_size;
+ if (f)
+ add_finalizer (result, f, s, n);
+
if (GATHER_STATISTICS)
{
size_t overhead = object_size - size;
unsigned long mask;
unsigned long offset;
- if (!p || !ggc_allocated_p (p))
+ if (!p)
return;
- /* Look up the page on which the object is alloced. . */
- entry = lookup_page_table_entry (p);
- gcc_assert (entry);
+ /* Look up the page on which the object is alloced. If it was not
+ GC allocated, gracefully bail out. */
+ entry = safe_lookup_page_table_entry (p);
+ if (!entry)
+ return;
/* Calculate the index of the object on the page; this is its bit
position in the in_use_p bitmap. Note that because a char* might
void
ggc_free (void *p)
{
+ if (in_gc)
+ return;
+
page_entry *pe = lookup_page_table_entry (p);
size_t order = pe->order;
size_t size = OBJECT_SIZE (order);
void
init_ggc (void)
{
+ static bool init_p = false;
unsigned order;
- G.pagesize = getpagesize();
+ if (init_p)
+ return;
+ init_p = true;
+
+ G.pagesize = getpagesize ();
G.lg_pagesize = exact_log2 (G.pagesize);
#ifdef HAVE_MMAP_DEV_ZERO
G.by_depth_max = INITIAL_PTE_COUNT;
G.by_depth = XNEWVEC (page_entry *, G.by_depth_max);
G.save_in_use = XNEWVEC (unsigned long *, G.by_depth_max);
+
+ /* Allocate space for the depth 0 finalizers. */
+ G.finalizers.safe_push (vNULL);
+ G.vec_finalizers.safe_push (vNULL);
+ gcc_assert (G.finalizers.length() == 1);
}
/* Merge the SAVE_IN_USE_P and IN_USE_P arrays in P so that IN_USE_P
}
}
+/* Check if any blocks with a registered finalizer have become unmarked. If so
+ run the finalizer and unregister it because the block is about to be freed.
+ Note that no garantee is made about what order finalizers will run in so
+ touching other objects in gc memory is extremely unwise. */
+
+static void
+ggc_handle_finalizers ()
+{
+ unsigned dlen = G.finalizers.length();
+ for (unsigned d = G.context_depth; d < dlen; ++d)
+ {
+ vec<finalizer> &v = G.finalizers[d];
+ unsigned length = v.length ();
+ for (unsigned int i = 0; i < length;)
+ {
+ finalizer &f = v[i];
+ if (!ggc_marked_p (f.addr ()))
+ {
+ f.call ();
+ v.unordered_remove (i);
+ length--;
+ }
+ else
+ i++;
+ }
+ }
+
+ gcc_assert (dlen == G.vec_finalizers.length());
+ for (unsigned d = G.context_depth; d < dlen; ++d)
+ {
+ vec<vec_finalizer> &vv = G.vec_finalizers[d];
+ unsigned length = vv.length ();
+ for (unsigned int i = 0; i < length;)
+ {
+ vec_finalizer &f = vv[i];
+ if (!ggc_marked_p (f.addr ()))
+ {
+ f.call ();
+ vv.unordered_remove (i);
+ length--;
+ }
+ else
+ i++;
+ }
+ }
+}
+
/* Free all empty pages. Partially empty pages need no attention
because the `mark' bit doubles as an `unused' bit. */
MAX (G.allocated_last_gc, (size_t)PARAM_VALUE (GGC_MIN_HEAPSIZE) * 1024);
float min_expand = allocated_last_gc * PARAM_VALUE (GGC_MIN_EXPAND) / 100;
-
if (G.allocated < allocated_last_gc + min_expand && !ggc_force_collect)
return;
invoke_plugin_callbacks (PLUGIN_GGC_START, NULL);
+ in_gc = true;
clear_marks ();
ggc_mark_roots ();
+ ggc_handle_finalizers ();
if (GATHER_STATISTICS)
ggc_prune_overhead_list ();
validate_free_objects ();
sweep_pages ();
+ in_gc = false;
G.allocated_last_gc = G.allocated;
invoke_plugin_callbacks (PLUGIN_GGC_END, NULL);
fprintf (G.debug_file, "END COLLECTING\n");
}
-/* Print allocation statistics. */
-#define SCALE(x) ((unsigned long) ((x) < 1024*10 \
- ? (x) \
- : ((x) < 1024*1024*10 \
- ? (x) / 1024 \
- : (x) / (1024*1024))))
-#define STAT_LABEL(x) ((x) < 1024*10 ? ' ' : ((x) < 1024*1024*10 ? 'k' : 'M'))
+/* Assume that all GGC memory is reachable and grow the limits for next collection.
+ With checking, trigger GGC so -Q compilation outputs how much of memory really is
+ reachable. */
+
+void
+ggc_grow (void)
+{
+ if (!flag_checking)
+ G.allocated_last_gc = MAX (G.allocated_last_gc,
+ G.allocated);
+ else
+ ggc_collect ();
+ if (!quiet_flag)
+ fprintf (stderr, " {GC start %luk} ", (unsigned long) G.allocated / 1024);
+}
void
ggc_print_statistics (void)
allocation. */
fprintf (stderr,
"Memory still allocated at the end of the compilation process\n");
- fprintf (stderr, "%-5s %10s %10s %10s\n",
+ fprintf (stderr, "%-8s %10s %10s %10s\n",
"Size", "Allocated", "Used", "Overhead");
for (i = 0; i < NUM_ORDERS; ++i)
{
overhead += (sizeof (page_entry) - sizeof (long)
+ BITMAP_SIZE (OBJECTS_IN_PAGE (p) + 1));
}
- fprintf (stderr, "%-5lu %10lu%c %10lu%c %10lu%c\n",
- (unsigned long) OBJECT_SIZE (i),
- SCALE (allocated), STAT_LABEL (allocated),
- SCALE (in_use), STAT_LABEL (in_use),
- SCALE (overhead), STAT_LABEL (overhead));
+ fprintf (stderr, "%-8" PRIu64 " " PRsa (10) " " PRsa (10) " "
+ PRsa (10) "\n",
+ (uint64_t)OBJECT_SIZE (i),
+ SIZE_AMOUNT (allocated),
+ SIZE_AMOUNT (in_use),
+ SIZE_AMOUNT (overhead));
total_overhead += overhead;
}
- fprintf (stderr, "%-5s %10lu%c %10lu%c %10lu%c\n", "Total",
- SCALE (G.bytes_mapped), STAT_LABEL (G.bytes_mapped),
- SCALE (G.allocated), STAT_LABEL(G.allocated),
- SCALE (total_overhead), STAT_LABEL (total_overhead));
+ fprintf (stderr, "%-8s " PRsa (10) " " PRsa (10) " " PRsa (10) "\n",
+ "Total",
+ SIZE_AMOUNT (G.bytes_mapped),
+ SIZE_AMOUNT (G.allocated),
+ SIZE_AMOUNT (total_overhead));
if (GATHER_STATISTICS)
{
- fprintf (stderr, "\nTotal allocations and overheads during the compilation process\n");
-
- fprintf (stderr, "Total Overhead: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_overhead);
- fprintf (stderr, "Total Allocated: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_allocated);
-
- fprintf (stderr, "Total Overhead under 32B: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_overhead_under32);
- fprintf (stderr, "Total Allocated under 32B: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_allocated_under32);
- fprintf (stderr, "Total Overhead under 64B: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_overhead_under64);
- fprintf (stderr, "Total Allocated under 64B: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_allocated_under64);
- fprintf (stderr, "Total Overhead under 128B: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_overhead_under128);
- fprintf (stderr, "Total Allocated under 128B: %10" HOST_LONG_LONG_FORMAT "d\n",
- G.stats.total_allocated_under128);
+ fprintf (stderr, "\nTotal allocations and overheads during "
+ "the compilation process\n");
+
+ fprintf (stderr, "Total Overhead: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_overhead));
+ fprintf (stderr, "Total Allocated: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_allocated));
+
+ fprintf (stderr, "Total Overhead under 32B: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_overhead_under32));
+ fprintf (stderr, "Total Allocated under 32B: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_allocated_under32));
+ fprintf (stderr, "Total Overhead under 64B: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_overhead_under64));
+ fprintf (stderr, "Total Allocated under 64B: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_allocated_under64));
+ fprintf (stderr, "Total Overhead under 128B: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_overhead_under128));
+ fprintf (stderr, "Total Allocated under 128B: "
+ PRsa (9) "\n",
+ SIZE_AMOUNT (G.stats.total_allocated_under128));
for (i = 0; i < NUM_ORDERS; i++)
if (G.stats.total_allocated_per_order[i])
{
- fprintf (stderr, "Total Overhead page size %7lu: %10" HOST_LONG_LONG_FORMAT "d\n",
- (unsigned long) OBJECT_SIZE (i),
- G.stats.total_overhead_per_order[i]);
- fprintf (stderr, "Total Allocated page size %7lu: %10" HOST_LONG_LONG_FORMAT "d\n",
- (unsigned long) OBJECT_SIZE (i),
- G.stats.total_allocated_per_order[i]);
+ fprintf (stderr, "Total Overhead page size %9" PRIu64 ": "
+ PRsa (9) "\n",
+ (uint64_t)OBJECT_SIZE (i),
+ SIZE_AMOUNT (G.stats.total_overhead_per_order[i]));
+ fprintf (stderr, "Total Allocated page size %9" PRIu64 ": "
+ PRsa (9) "\n",
+ (uint64_t)OBJECT_SIZE (i),
+ SIZE_AMOUNT (G.stats.total_allocated_per_order[i]));
}
}
}
}
if (fwrite (x, size, 1, f) != 1)
- fatal_error ("can%'t write PCH file: %m");
+ fatal_error (input_location, "cannot write PCH file: %m");
/* If SIZE is not the same as OBJECT_SIZE(order), then we need to pad the
object out to OBJECT_SIZE(order). This happens for strings. */
if (size != OBJECT_SIZE (order))
{
- unsigned padding = OBJECT_SIZE(order) - size;
+ unsigned padding = OBJECT_SIZE (order) - size;
/* To speed small writes, we use a nulled-out array that's larger
than most padding requests as the source for our null bytes. This
permits us to do the padding with fwrite() rather than fseek(), and
limits the chance the OS may try to flush any outstanding writes. */
- if (padding <= sizeof(emptyBytes))
+ if (padding <= sizeof (emptyBytes))
{
if (fwrite (emptyBytes, 1, padding, f) != padding)
- fatal_error ("can%'t write PCH file");
+ fatal_error (input_location, "cannot write PCH file");
}
else
{
/* Larger than our buffer? Just default to fseek. */
if (fseek (f, padding, SEEK_CUR) != 0)
- fatal_error ("can%'t write PCH file");
+ fatal_error (input_location, "cannot write PCH file");
}
}
&& fseek (f, ROUND_UP_VALUE (d->d.totals[order] * OBJECT_SIZE (order),
G.pagesize),
SEEK_CUR) != 0)
- fatal_error ("can%'t write PCH file: %m");
+ fatal_error (input_location, "cannot write PCH file: %m");
}
void
ggc_pch_finish (struct ggc_pch_data *d, FILE *f)
{
if (fwrite (&d->d, sizeof (d->d), 1, f) != 1)
- fatal_error ("can%'t write PCH file: %m");
+ fatal_error (input_location, "cannot write PCH file: %m");
free (d);
}
static void
move_ptes_to_front (int count_old_page_tables, int count_new_page_tables)
{
- unsigned i;
-
/* First, we swap the new entries to the front of the varrays. */
page_entry **new_by_depth;
unsigned long **new_save_in_use;
G.save_in_use = new_save_in_use;
/* Now update all the index_by_depth fields. */
- for (i = G.by_depth_in_use; i > 0; --i)
+ for (unsigned i = G.by_depth_in_use; i--;)
{
- page_entry *p = G.by_depth[i-1];
- p->index_by_depth = i-1;
+ page_entry *p = G.by_depth[i];
+ p->index_by_depth = i;
}
/* And last, we update the depth pointers in G.depth. The first
#endif
/* Since we free all the allocated objects, the free list becomes
useless. Validate it now, which will also clear it. */
- validate_free_objects();
+ validate_free_objects ();
/* No object read from a PCH file should ever be freed. So, set the
context depth to 1, and set the depth of all the currently-allocated
pages to be 1 too. PCH pages will have depth 0. */
gcc_assert (!G.context_depth);
G.context_depth = 1;
+ /* Allocate space for the depth 1 finalizers. */
+ G.finalizers.safe_push (vNULL);
+ G.vec_finalizers.safe_push (vNULL);
+ gcc_assert (G.finalizers.length() == 2);
for (i = 0; i < NUM_ORDERS; i++)
{
page_entry *p;
/* Allocate the appropriate page-table entries for the pages read from
the PCH file. */
if (fread (&d, sizeof (d), 1, f) != 1)
- fatal_error ("can%'t read PCH file: %m");
+ fatal_error (input_location, "cannot read PCH file: %m");
for (i = 0; i < NUM_ORDERS; i++)
{
projects / thirdparty / gcc.git / blobdiff