print_size("Current config:", rmemsize(config_pool));
struct resmem total = rmemsize(&root_pool);
#ifdef HAVE_MMAP
- int pk = atomic_load_explicit(&pages_kept, memory_order_relaxed)
- + atomic_load_explicit(&pages_kept_locally, memory_order_relaxed)
- + atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed);
- print_size("Standby memory:", (struct resmem) { .overhead = page_size * pk });
- total.overhead += page_size * pk;
- uint pages_standby = *pages_kept + pages_kept_cold_index;
- print_size("Standby memory:", (struct resmem) { .overhead = page_size * pages_standby });
- total.overhead += page_size * pages_standby;
++ uint hot_pages = atomic_load_explicit(&pages_kept, memory_order_relaxed)
++ + atomic_load_explicit(&pages_kept_locally, memory_order_relaxed);
++ uint cold_pages_index = atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed);
++ print_size("Standby memory:", (struct resmem) { .overhead = page_size * (hot_pages + cold_pages_index) });
++ total.overhead += page_size * (hot_pages + cold_pages_index);
#endif
+
print_size("Total:", total);
+ cli_msg(-1018, "");
- uint pages_active = pages_total - *pages_kept - pages_kept_cold;
+#ifdef HAVE_MMAP
- struct size_args cold = get_size_args(atomic_load_explicit(&pages_kept_cold, memory_order_relaxed) * page_size);
- cli_msg(-1018, "%-23s " SIZE_FORMAT, "Cold memory:", SIZE_ARGS(cold));
++ uint cold_pages = atomic_load_explicit(&pages_kept_cold, memory_order_relaxed);
++ uint pages_total_loc = atomic_load_explicit(&pages_total, memory_order_relaxed);
++ uint pages_active = pages_total_loc - hot_pages - cold_pages_index - cold_pages;
++
+ struct size_args active = get_size_args(page_size * pages_active);
- struct size_args kept = get_size_args(page_size * *pages_kept);
- struct size_args cold = get_size_args(page_size * pages_kept_cold);
++ struct size_args kept = get_size_args(page_size * (hot_pages + cold_pages_index));
++ struct size_args cold = get_size_args(page_size * cold_pages);
+
+ cli_msg(-1018, "%-17s " SIZE_FORMAT, "Active pages:", SIZE_ARGS(active));
+ cli_msg(-1018, "%-17s " SIZE_FORMAT, "Kept free pages:", SIZE_ARGS(kept));
+ cli_msg(-1018, "%-17s " SIZE_FORMAT, "Cold free pages:", SIZE_ARGS(cold));
+
+#endif
cli_msg(0, "");
}
long page_size = 0;
#ifdef HAVE_MMAP
-#define KEEP_PAGES_MAIN_MAX 256
-#define KEEP_PAGES_MAIN_MIN 8
-#define CLEANUP_PAGES_BULK 256
+# define KEEP_PAGES_MAX 16384
+# define KEEP_PAGES_MIN 32
+# define KEEP_PAGES_MAX_LOCAL 128
+# define ALLOC_PAGES_AT_ONCE 32
+
+ STATIC_ASSERT(KEEP_PAGES_MIN * 4 < KEEP_PAGES_MAX);
+ STATIC_ASSERT(ALLOC_PAGES_AT_ONCE < KEEP_PAGES_MAX_LOCAL);
+
+ static bool use_fake = 0;
+ static bool initialized = 0;
+
+# define PROTECT_PAGE(pg)
+# define UNPROTECT_PAGE(pg)
+
+# if DEBUGGING
+# ifdef ENABLE_EXPENSIVE_CHECKS
+# undef PROTECT_PAGE
+# undef UNPROTECT_PAGE
+# define PROTECT_PAGE(pg) mprotect((pg), page_size, PROT_READ)
+# define UNPROTECT_PAGE(pg) mprotect((pg), page_size, PROT_READ | PROT_WRITE)
+# endif
+
+# define AJSIZE 16384
+
+ static struct alloc_journal {
+ void *fp;
+ void *next;
+ u16 pos;
+ u16 type;
+ uint thread_id;
+ } alloc_journal[AJSIZE];
+
+ _Thread_local int alloc_journal_local_pos = -1;
+ _Atomic int alloc_journal_pos = 0;
+
+# define AJT_ALLOC_LOCAL_HOT 1
+# define AJT_ALLOC_GLOBAL_HOT 2
+# define AJT_ALLOC_COLD_STD 3
+# define AJT_ALLOC_COLD_KEEPER 4
+# define AJT_ALLOC_MMAP 5
+
+# define AJT_FREE_LOCAL_HOT 0x11
+# define AJT_FREE_GLOBAL_HOT 0x12
+
+# define AJT_CLEANUP_NOTHING 0xc0
+# define AJT_CLEANUP_COLD_STD 0xc3
+# define AJT_CLEANUP_COLD_KEEPER 0xc4
+# define AJT_CLEANUP_BEGIN 0xcb
+# define AJT_CLEANUP_END 0xce
+
+# define AJT_FLUSH_LOCAL_BEGIN 0xfb
+# define AJT_FLUSH_LOCAL_END 0xfe
+# define AJT_SCHEDULE_CLEANUP 0xff
+
+ static void
+ ajlog(void *fp, void *next, u16 pos, u16 type)
+ {
+ alloc_journal[(alloc_journal_local_pos = atomic_fetch_add_explicit(&alloc_journal_pos, 1, memory_order_relaxed)) % AJSIZE] = (struct alloc_journal) {
+ .fp = fp,
+ .next = next,
+ .pos = pos,
+ .type = type,
+ .thread_id = THIS_THREAD_ID,
+ };
+ }
-STATIC_ASSERT(KEEP_PAGES_MAIN_MIN * 4 < KEEP_PAGES_MAIN_MAX);
+ struct free_page {
+ node unused[42];
+ struct free_page * _Atomic next;
+ };
+# else /* ! DEBUGGING */
-static bool use_fake = 0;
+# define ajlog(...)
-#if DEBUGGING
-struct free_page {
- node unused[42];
- node n;
-};
-#else
-struct free_page {
- node n;
-};
-#endif
+ struct free_page {
+ struct free_page * _Atomic next;
+ };
-#define EP_POS_MAX ((page_size - OFFSETOF(struct empty_pages, pages)) / sizeof (void *))
+# endif
-struct empty_pages {
- node n;
- uint pos;
- void *pages[0];
-};
+# define WRITE_NEXT(pg, val) do { UNPROTECT_PAGE((pg)); (pg)->next = (val); PROTECT_PAGE((pg)); } while (0)
-struct free_pages {
- list pages; /* List of (struct free_page) keeping free pages without releasing them (hot) */
- list empty; /* List of (struct empty_pages) keeping invalidated pages mapped for us (cold) */
- u16 min, max; /* Minimal and maximal number of free pages kept */
- uint cnt; /* Number of free pages in list */
- event cleanup;
-};
+# define EP_POS_MAX ((page_size - OFFSETOF(struct empty_pages, pages)) / sizeof (void *))
-static void global_free_pages_cleanup_event(void *);
-static void *alloc_cold_page(void);
+ struct empty_pages {
+ struct empty_pages *next;
+ uint pos;
+ void *pages[0];
+ };
-static struct free_pages global_free_pages = {
- .min = KEEP_PAGES_MAIN_MIN,
- .max = KEEP_PAGES_MAIN_MAX,
- .cleanup = { .hook = global_free_pages_cleanup_event },
-};
+ static DOMAIN(resource) empty_pages_domain;
+ static struct empty_pages *empty_pages = NULL;
+ _Atomic int pages_kept_cold = 0;
+ _Atomic int pages_kept_cold_index = 0;
++ _Atomic int pages_total = 0;
-uint *pages_kept = &global_free_pages.cnt;
-uint pages_kept_cold, pages_kept_cold_index, pages_total;
+ static struct free_page * _Atomic page_stack = NULL;
+ static _Thread_local struct free_page * local_page_stack = NULL;
+ static struct free_page page_stack_blocked;
-static void *
-alloc_sys_page(void)
-{
- pages_total++;
+ /* Try to replace the page stack head with a cork, until it succeeds. */
+# define PAGE_STACK_GET ({ \
+ struct free_page *fp; \
+ while ((fp = atomic_exchange_explicit(&page_stack, &page_stack_blocked, memory_order_acq_rel)) == &page_stack_blocked) birdloop_yield(); \
+ fp; })
+ /* Reinstate the stack with another value */
+# define PAGE_STACK_PUT(val) ASSERT_DIE(atomic_exchange_explicit(&page_stack, (val), memory_order_acq_rel) == &page_stack_blocked)
- void *ptr = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ static void page_cleanup(void *);
+ static event page_cleanup_event = { .hook = page_cleanup, };
+# define SCHEDULE_CLEANUP do if (initialized && !shutting_down) ev_send(&global_event_list, &page_cleanup_event); while (0)
- if (ptr == MAP_FAILED)
- die("mmap(%ld) failed: %m", (s64) page_size);
+ _Atomic int pages_kept = 0;
+ _Atomic int pages_kept_locally = 0;
+ static _Thread_local int pages_kept_here = 0;
- return ptr;
-}
+ static void *
+ alloc_sys_page(void)
+ {
+ void *ptr = mmap(NULL, page_size * ALLOC_PAGES_AT_ONCE, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+ if (ptr == MAP_FAILED)
+ die("mmap(%ld) failed: %m", (s64) page_size);
+
++ atomic_fetch_add_explicit(&pages_total, ALLOC_PAGES_AT_ONCE, memory_order_acq_rel);
+ return ptr;
+ }
-extern int shutting_down; /* Shutdown requested. */
+ extern int shutting_down; /* Shutdown requested. */
#else // ! HAVE_MMAP
-#define use_fake 1
+# define use_fake 1
#endif
+#define ALLOC_TRACE(fmt...) do { \
+ if (atomic_load_explicit(&global_runtime, memory_order_relaxed)->latency_debug & DL_ALLOCATOR) log(L_TRACE "Allocator: " fmt, ##fmt); } while (0)
+
void *
alloc_page(void)
{
/* If the system page allocator is goofy, we use posix_memalign to get aligned blocks of memory. */
if (use_fake)
{
- pages_total++;
++ atomic_fetch_add_explicit(&pages_total, 1, memory_order_acq_rel);
void *ptr = NULL;
int err = posix_memalign(&ptr, page_size, page_size);
/* If the system page allocator is goofy, we just free the block and care no more. */
if (use_fake)
{
- pages_total--;
++ atomic_fetch_sub_explicit(&pages_total, 1, memory_order_acq_rel);
free(ptr);
return;
}
for (uint i=0; i<ep->pos; i++)
RDUMP(" %p\n", ep->pages[i]);
}
+ UNLOCK_DOMAIN(resource, empty_pages_domain);
+ RDUMP("This request: %p\n", dreq);
#endif
}
projects / thirdparty / bird.git / commitdiff
