Allocator now reports cold pages

diff --git a/conf/conf.h b/conf/conf.h
index 7ed22afc165ee56aafb59e0c0046fb3e29a86668..9a06ad4df45003e86de01e65dbab69c2bcb667c9 100644 (file)
--- a/conf/conf.h
+++ b/conf/conf.h
@@ -51,6 +51,7 @@ struct config {
     DL_PING = 1,
     DL_WAKEUP = 2,
     DL_SCHEDULING = 4,
+    DL_ALLOCATOR = 8,
     DL_SOCKETS = 0x10,
     DL_EVENTS = 0x20,
     DL_TIMERS = 0x40,

diff --git a/lib/resource.h b/lib/resource.h
index 6b225c768257ef90df86a29152d7eb21c48d4505..2d540723dd1f53a69500ef9f4bbed30cd258cc5e 100644 (file)
--- a/lib/resource.h
+++ b/lib/resource.h
@@ -148,6 +148,8 @@ void buffer_realloc(void **buf, unsigned *size, unsigned need, unsigned item_siz
 extern long page_size;
 extern _Atomic int pages_kept;
 extern _Atomic int pages_kept_locally;
+extern _Atomic int pages_kept_cold;
+extern _Atomic int pages_kept_cold_index;
 void *alloc_page(void);
 void free_page(void *);
 void flush_local_pages(void);

diff --git a/nest/cmds.c b/nest/cmds.c
index b3729a5b7f1baa4a477b2729f810a7f736fee03d..b55d2c2354e024d3885c6556e6f00687b633c3a9 100644 (file)
--- a/nest/cmds.c
+++ b/nest/cmds.c
@@ -125,11 +125,18 @@ cmd_show_memory(void)
   struct resmem total = rmemsize(&root_pool);
 #ifdef HAVE_MMAP
   int pk  = atomic_load_explicit(&pages_kept, memory_order_relaxed)
-         + atomic_load_explicit(&pages_kept_locally, memory_order_relaxed);
+         + atomic_load_explicit(&pages_kept_locally, memory_order_relaxed)
+         + atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed);
   print_size("Standby memory:", (struct resmem) { .overhead = page_size * pk });
   total.overhead += page_size * pk;
 #endif
+
   print_size("Total:", total);
+
+#ifdef HAVE_MMAP
+  struct size_args cold = get_size_args(atomic_load_explicit(&pages_kept_cold, memory_order_relaxed) * page_size);
+  cli_msg(-1018, "%-23s " SIZE_FORMAT, "Cold memory:", SIZE_ARGS(cold));
+#endif
   cli_msg(0, "");
 }
 

diff --git a/sysdep/unix/alloc.c b/sysdep/unix/alloc.c
index ef383f3653810db9e7972e6d4d516bf752ebd1a2..029609561ab86fcc3357f4d94b7d66f5b147e7b2 100644 (file)
--- a/sysdep/unix/alloc.c
+++ b/sysdep/unix/alloc.c
@@ -12,6 +12,8 @@
 #include "lib/event.h"
 #include "lib/io-loop.h"
 
+#include "conf/conf.h"
+
 #include <errno.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -122,6 +124,8 @@ long page_size = 0;
 
   static DOMAIN(resource) empty_pages_domain;
   static struct empty_pages *empty_pages = NULL;
+  _Atomic int pages_kept_cold = 0;
+  _Atomic int pages_kept_cold_index = 0;
 
   static struct free_page * _Atomic page_stack = NULL;
   static _Thread_local struct free_page * local_page_stack = NULL;
@@ -160,6 +164,9 @@ long page_size = 0;
 # define use_fake  1
 #endif
 
+#define ALLOC_TRACE(fmt...) do { \
+  if (atomic_load_explicit(&global_runtime, memory_order_relaxed)->latency_debug & DL_ALLOCATOR) log(L_TRACE "Allocator: " fmt, ##fmt); } while (0)
+
 void *
 alloc_page(void)
 {
@@ -219,6 +226,7 @@ alloc_page(void)
       PROTECT_PAGE(empty_pages);
       UNPROTECT_PAGE(fp);
       ajlog(fp, empty_pages, empty_pages->pos, AJT_ALLOC_COLD_STD);
+      atomic_fetch_sub_explicit(&pages_kept_cold, 1, memory_order_relaxed);
     }
     else
     {
@@ -226,6 +234,9 @@ alloc_page(void)
       fp = (struct free_page *) empty_pages;
       empty_pages = empty_pages->next;
       ajlog(fp, empty_pages, 0, AJT_ALLOC_COLD_KEEPER);
+      atomic_fetch_sub_explicit(&pages_kept_cold_index, 1, memory_order_relaxed);
+      if (!empty_pages)
+       ALLOC_TRACE("Taken last page from cold storage");
     }
   }
   UNLOCK_DOMAIN(resource, empty_pages_domain);
@@ -339,25 +350,29 @@ page_cleanup(void *_ UNUSED)
   if (shutting_down)
     return;
 
-  ajlog(NULL, NULL, 0, AJT_CLEANUP_BEGIN);
-
-  /* Prevent contention */
-  struct free_page *stack = PAGE_STACK_GET;
-
-  /* Always replace by zero */
-  PAGE_STACK_PUT(NULL);
-
-  if (!stack)
-  {
-    ajlog(NULL, NULL, 0, AJT_CLEANUP_NOTHING);
+  /* Pages allocated inbetween */
+  uint pk = atomic_load_explicit(&pages_kept, memory_order_relaxed);
+  if (pk < KEEP_PAGES_MAX)
     return;
-  }
 
+  /* Walk the pages */
+  ajlog(NULL, NULL, 0, AJT_CLEANUP_BEGIN);
+  uint count = 0;
   do {
-    struct free_page *fp = stack;
-    stack = atomic_load_explicit(&fp->next, memory_order_relaxed);
+    /* Get next hot page */
+    struct free_page *fp = PAGE_STACK_GET;
+    if (!fp) {
+      PAGE_STACK_PUT(NULL);
+      ajlog(NULL, NULL, 0, AJT_CLEANUP_END);
+      return;
+    }
+
+    /* Reinstate the stack with the next page in list */
+    PAGE_STACK_PUT(atomic_load_explicit(&fp->next, memory_order_relaxed));
 
+    /* Cold pages are locked */
     LOCK_DOMAIN(resource, empty_pages_domain);
+
     /* Empty pages are stored as pointers. To store them, we need a pointer block. */
     if (!empty_pages || (empty_pages->pos == EP_POS_MAX))
     {
@@ -370,6 +385,7 @@ page_cleanup(void *_ UNUSED)
       PROTECT_PAGE(ep);
       empty_pages = ep;
       ajlog(empty_pages, empty_pages->next, 0, AJT_CLEANUP_COLD_KEEPER);
+      atomic_fetch_add_explicit(&pages_kept_cold_index, 1, memory_order_relaxed);
     }
     else
     {
@@ -389,20 +405,18 @@ page_cleanup(void *_ UNUSED)
            ) < 0)
        bug("madvise(%p) failed: %m", fp);
       ajlog(fp, empty_pages, empty_pages->pos, AJT_CLEANUP_COLD_STD);
+      atomic_fetch_add_explicit(&pages_kept_cold, 1, memory_order_relaxed);
     }
     UNLOCK_DOMAIN(resource, empty_pages_domain);
+    count++;
   }
-  while ((atomic_fetch_sub_explicit(&pages_kept, 1, memory_order_relaxed) >= KEEP_PAGES_MAX / 2) && stack);
+  while (atomic_fetch_sub_explicit(&pages_kept, 1, memory_order_relaxed) >= KEEP_PAGES_MAX / 2);
 
-  while (stack)
-  {
-    struct free_page *f = stack;
-    stack = atomic_load_explicit(&f->next, memory_order_acquire);
-    UNPROTECT_PAGE(f);
-    free_page(f);
+  ALLOC_TRACE("Moved %u pages to cold storage, now %u cold, %u index", count,
+      atomic_load_explicit(&pages_kept_cold, memory_order_relaxed),
+      atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed)
+      );
 
-    atomic_fetch_sub_explicit(&pages_kept, 1, memory_order_relaxed);
-  }
   ajlog(NULL, NULL, 0, AJT_CLEANUP_END);
 }
 #endif

diff --git a/sysdep/unix/config.Y b/sysdep/unix/config.Y
index 1b72007c675a01e749e4dc124ae2eb8af9beaacc..46365c85051157b3d6bd115708d6bf5e165fac91 100644 (file)
--- a/sysdep/unix/config.Y
+++ b/sysdep/unix/config.Y
@@ -19,7 +19,7 @@ CF_DECLS
 
 CF_KEYWORDS(LOG, SYSLOG, ALL, DEBUG, TRACE, INFO, REMOTE, WARNING, ERROR, AUTH, FATAL, BUG, STDERR, SOFT, UDP, PORT)
 CF_KEYWORDS(NAME, CONFIRM, UNDO, CHECK, TIMEOUT, DEBUG, LATENCY, LIMIT, WATCHDOG, WARNING, STATUS)
-CF_KEYWORDS(PING, WAKEUP, SOCKETS, SCHEDULING, EVENTS, TIMERS)
+CF_KEYWORDS(PING, WAKEUP, SOCKETS, SCHEDULING, EVENTS, TIMERS, ALLOCATOR)
 CF_KEYWORDS(GRACEFUL, RESTART, FIXED)
 
 %type <i> log_mask log_mask_list log_cat cfg_timeout debug_unix latency_debug_mask latency_debug_flag latency_debug_list
@@ -151,6 +151,7 @@ latency_debug_flag:
  | WAKEUP      { $$ = DL_WAKEUP; }
  | SOCKETS     { $$ = DL_SOCKETS; }
  | SCHEDULING  { $$ = DL_SCHEDULING; }
+ | ALLOCATOR   { $$ = DL_ALLOCATOR; }
  | EVENTS      { $$ = DL_EVENTS; }
  | TIMERS      { $$ = DL_TIMERS; }
  ;