From: Maria Matejka <mq@ucw.cz>
Date: Fri, 13 Dec 2024 17:40:11 +0000 (+0100)
Subject: Merge commit '707cad61' into thread-next
X-Git-Tag: v3.0.0~15
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=93621ed9f4441fb540726e06fbf7406f37f30ac6;p=thirdparty%2Fbird.git

Merge commit '707cad61' into thread-next
---

93621ed9f4441fb540726e06fbf7406f37f30ac6
diff --cc lib/resource.h
index 7961e4f75,1e8e2d2ce..8baa693c6
--- a/lib/resource.h
+++ b/lib/resource.h
@@@ -147,15 -111,9 +147,16 @@@ void sl_delete(slab *)
  void buffer_realloc(void **buf, unsigned *size, unsigned need, unsigned item_size);
  
  /* Allocator of whole pages; for use in slabs and other high-level allocators. */
 +#define PAGE_HEAD(x)	((void *) (((uintptr_t) (x)) & ~(page_size-1)))
  extern long page_size;
 +extern _Atomic int pages_kept;
 +extern _Atomic int pages_kept_locally;
 +extern _Atomic int pages_kept_cold;
 +extern _Atomic int pages_kept_cold_index;
++extern _Atomic int pages_total;
  void *alloc_page(void);
  void free_page(void *);
 +void flush_local_pages(void);
  
  void resource_sys_init(void);
  
diff --cc nest/cmds.c
index a596773c9,16862eb7b..c46c94f5e
--- a/nest/cmds.c
+++ b/nest/cmds.c
@@@ -123,19 -124,22 +123,30 @@@ cmd_show_memory(void
    print_size("Current config:", rmemsize(config_pool));
    struct resmem total = rmemsize(&root_pool);
  #ifdef HAVE_MMAP
-   int pk  = atomic_load_explicit(&pages_kept, memory_order_relaxed)
- 	  + atomic_load_explicit(&pages_kept_locally, memory_order_relaxed)
- 	  + atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed);
-   print_size("Standby memory:", (struct resmem) { .overhead = page_size * pk });
-   total.overhead += page_size * pk;
 -  uint pages_standby = *pages_kept + pages_kept_cold_index;
 -  print_size("Standby memory:", (struct resmem) { .overhead = page_size * pages_standby });
 -  total.overhead += page_size * pages_standby;
++  uint hot_pages = atomic_load_explicit(&pages_kept, memory_order_relaxed)
++		+ atomic_load_explicit(&pages_kept_locally, memory_order_relaxed);
++  uint cold_pages_index = atomic_load_explicit(&pages_kept_cold_index, memory_order_relaxed);
++  print_size("Standby memory:", (struct resmem) { .overhead = page_size * (hot_pages + cold_pages_index) });
++  total.overhead += page_size * (hot_pages + cold_pages_index);
  #endif
 +
    print_size("Total:", total);
+   cli_msg(-1018, "");
  
 -  uint pages_active = pages_total - *pages_kept - pages_kept_cold;
 +#ifdef HAVE_MMAP
-   struct size_args cold = get_size_args(atomic_load_explicit(&pages_kept_cold, memory_order_relaxed) * page_size);
-   cli_msg(-1018, "%-23s " SIZE_FORMAT, "Cold memory:", SIZE_ARGS(cold));
++  uint cold_pages = atomic_load_explicit(&pages_kept_cold, memory_order_relaxed);
++  uint pages_total_loc = atomic_load_explicit(&pages_total, memory_order_relaxed);
++  uint pages_active = pages_total_loc - hot_pages - cold_pages_index - cold_pages;
++  
+   struct size_args active = get_size_args(page_size * pages_active);
 -  struct size_args kept = get_size_args(page_size * *pages_kept);
 -  struct size_args cold = get_size_args(page_size * pages_kept_cold);
++  struct size_args kept = get_size_args(page_size * (hot_pages + cold_pages_index));
++  struct size_args cold = get_size_args(page_size * cold_pages);
+ 
+   cli_msg(-1018, "%-17s " SIZE_FORMAT, "Active pages:", SIZE_ARGS(active));
+   cli_msg(-1018, "%-17s " SIZE_FORMAT, "Kept free pages:", SIZE_ARGS(kept));
+   cli_msg(-1018, "%-17s " SIZE_FORMAT, "Cold free pages:", SIZE_ARGS(cold));
+ 
 +#endif
    cli_msg(0, "");
  }
  
diff --cc sysdep/unix/alloc.c
index cbf873f92,7f0aabd46..f9c6bfcad
--- a/sysdep/unix/alloc.c
+++ b/sysdep/unix/alloc.c
@@@ -32,147 -29,79 +32,150 @@@
  long page_size = 0;
  
  #ifdef HAVE_MMAP
 -#define KEEP_PAGES_MAIN_MAX	256
 -#define KEEP_PAGES_MAIN_MIN	8
 -#define CLEANUP_PAGES_BULK	256
 +# define KEEP_PAGES_MAX	16384
 +# define KEEP_PAGES_MIN	32
 +# define KEEP_PAGES_MAX_LOCAL	128
 +# define ALLOC_PAGES_AT_ONCE	32
 +
 +  STATIC_ASSERT(KEEP_PAGES_MIN * 4 < KEEP_PAGES_MAX);
 +  STATIC_ASSERT(ALLOC_PAGES_AT_ONCE < KEEP_PAGES_MAX_LOCAL);
 +
 +  static bool use_fake = 0;
 +  static bool initialized = 0;
 +
 +# define PROTECT_PAGE(pg)
 +# define UNPROTECT_PAGE(pg)
 +
 +# if DEBUGGING
 +#   ifdef ENABLE_EXPENSIVE_CHECKS
 +#     undef PROTECT_PAGE
 +#     undef UNPROTECT_PAGE
 +#     define PROTECT_PAGE(pg)	mprotect((pg), page_size, PROT_READ)
 +#     define UNPROTECT_PAGE(pg)	mprotect((pg), page_size, PROT_READ | PROT_WRITE)
 +#   endif
 +
 +#   define AJSIZE	16384
 +
 +    static struct alloc_journal {
 +      void *fp;
 +      void *next;
 +      u16 pos;
 +      u16 type;
 +      uint thread_id;
 +    } alloc_journal[AJSIZE];
 +
 +    _Thread_local int alloc_journal_local_pos = -1;
 +    _Atomic int alloc_journal_pos = 0;
 +
 +#   define AJT_ALLOC_LOCAL_HOT		1
 +#   define AJT_ALLOC_GLOBAL_HOT		2
 +#   define AJT_ALLOC_COLD_STD		3
 +#   define AJT_ALLOC_COLD_KEEPER	4
 +#   define AJT_ALLOC_MMAP		5
 +
 +#   define AJT_FREE_LOCAL_HOT		0x11
 +#   define AJT_FREE_GLOBAL_HOT		0x12
 +
 +#   define AJT_CLEANUP_NOTHING		0xc0
 +#   define AJT_CLEANUP_COLD_STD		0xc3
 +#   define AJT_CLEANUP_COLD_KEEPER	0xc4
 +#   define AJT_CLEANUP_BEGIN		0xcb
 +#   define AJT_CLEANUP_END		0xce
 +
 +#   define AJT_FLUSH_LOCAL_BEGIN	0xfb
 +#   define AJT_FLUSH_LOCAL_END		0xfe
 +#   define AJT_SCHEDULE_CLEANUP		0xff
 +
 +    static void
 +    ajlog(void *fp, void *next, u16 pos, u16 type)
 +    {
 +      alloc_journal[(alloc_journal_local_pos = atomic_fetch_add_explicit(&alloc_journal_pos, 1, memory_order_relaxed)) % AJSIZE] = (struct alloc_journal) {
 +	.fp = fp,
 +	.next = next,
 +	.pos = pos,
 +	.type = type,
 +	.thread_id = THIS_THREAD_ID,
 +      };
 +    }
  
 -STATIC_ASSERT(KEEP_PAGES_MAIN_MIN * 4 < KEEP_PAGES_MAIN_MAX);
 +    struct free_page {
 +      node unused[42];
 +      struct free_page * _Atomic next;
 +    };
 +# else /* ! DEBUGGING */
  
 -static bool use_fake = 0;
 +#   define ajlog(...)
  
 -#if DEBUGGING
 -struct free_page {
 -  node unused[42];
 -  node n;
 -};
 -#else
 -struct free_page {
 -  node n;
 -};
 -#endif
 +    struct free_page {
 +      struct free_page * _Atomic next;
 +    };
  
 -#define EP_POS_MAX	((page_size - OFFSETOF(struct empty_pages, pages)) / sizeof (void *))
 +# endif
  
 -struct empty_pages {
 -  node n;
 -  uint pos;
 -  void *pages[0];
 -};
 +# define WRITE_NEXT(pg, val)	do { UNPROTECT_PAGE((pg)); (pg)->next = (val); PROTECT_PAGE((pg)); } while (0)
  
 -struct free_pages {
 -  list pages;		/* List of (struct free_page) keeping free pages without releasing them (hot) */
 -  list empty;		/* List of (struct empty_pages) keeping invalidated pages mapped for us (cold) */
 -  u16 min, max;		/* Minimal and maximal number of free pages kept */
 -  uint cnt;		/* Number of free pages in list */
 -  event cleanup;
 -};
 +# define EP_POS_MAX	((page_size - OFFSETOF(struct empty_pages, pages)) / sizeof (void *))
  
 -static void global_free_pages_cleanup_event(void *);
 -static void *alloc_cold_page(void);
 +  struct empty_pages {
 +    struct empty_pages *next;
 +    uint pos;
 +    void *pages[0];
 +  };
  
 -static struct free_pages global_free_pages = {
 -  .min = KEEP_PAGES_MAIN_MIN,
 -  .max = KEEP_PAGES_MAIN_MAX,
 -  .cleanup = { .hook = global_free_pages_cleanup_event },
 -};
 +  static DOMAIN(resource) empty_pages_domain;
 +  static struct empty_pages *empty_pages = NULL;
 +  _Atomic int pages_kept_cold = 0;
 +  _Atomic int pages_kept_cold_index = 0;
++  _Atomic int pages_total = 0;
  
 -uint *pages_kept = &global_free_pages.cnt;
 -uint pages_kept_cold, pages_kept_cold_index, pages_total;
 +  static struct free_page * _Atomic page_stack = NULL;
 +  static _Thread_local struct free_page * local_page_stack = NULL;
 +  static struct free_page page_stack_blocked;
  
 -static void *
 -alloc_sys_page(void)
 -{
 -  pages_total++;
 +  /* Try to replace the page stack head with a cork, until it succeeds. */
 +# define PAGE_STACK_GET	({ \
 +    struct free_page *fp; \
 +    while ((fp = atomic_exchange_explicit(&page_stack, &page_stack_blocked, memory_order_acq_rel)) == &page_stack_blocked) birdloop_yield(); \
 +    fp; })
 +  /* Reinstate the stack with another value */
 +# define PAGE_STACK_PUT(val)	ASSERT_DIE(atomic_exchange_explicit(&page_stack, (val), memory_order_acq_rel) == &page_stack_blocked)
  
 -  void *ptr = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 +  static void page_cleanup(void *);
 +  static event page_cleanup_event = { .hook = page_cleanup, };
 +# define SCHEDULE_CLEANUP  do if (initialized && !shutting_down) ev_send(&global_event_list, &page_cleanup_event); while (0)
  
 -  if (ptr == MAP_FAILED)
 -    die("mmap(%ld) failed: %m", (s64) page_size);
 +  _Atomic int pages_kept = 0;
 +  _Atomic int pages_kept_locally = 0;
 +  static _Thread_local int pages_kept_here = 0;
  
 -  return ptr;
 -}
 +  static void *
 +  alloc_sys_page(void)
 +  {
 +    void *ptr = mmap(NULL, page_size * ALLOC_PAGES_AT_ONCE, PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 +
 +    if (ptr == MAP_FAILED)
 +      die("mmap(%ld) failed: %m", (s64) page_size);
 +
++    atomic_fetch_add_explicit(&pages_total, ALLOC_PAGES_AT_ONCE, memory_order_acq_rel);
 +    return ptr;
 +  }
  
 -extern int shutting_down; /* Shutdown requested. */
 +  extern int shutting_down; /* Shutdown requested. */
  
  #else // ! HAVE_MMAP
 -#define use_fake  1
 +# define use_fake  1
  #endif
  
 +#define ALLOC_TRACE(fmt...) do { \
 +  if (atomic_load_explicit(&global_runtime, memory_order_relaxed)->latency_debug & DL_ALLOCATOR) log(L_TRACE "Allocator: " fmt, ##fmt); } while (0)
 +
  void *
  alloc_page(void)
  {
    /* If the system page allocator is goofy, we use posix_memalign to get aligned blocks of memory. */
    if (use_fake)
    {
 -    pages_total++;
++    atomic_fetch_add_explicit(&pages_total, 1, memory_order_acq_rel);
      void *ptr = NULL;
      int err = posix_memalign(&ptr, page_size, page_size);
  
@@@ -271,6 -164,7 +274,7 @@@ free_page(void *ptr
    /* If the system page allocator is goofy, we just free the block and care no more. */
    if (use_fake)
    {
 -    pages_total--;
++    atomic_fetch_sub_explicit(&pages_total, 1, memory_order_acq_rel);
      free(ptr);
      return;
    }
@@@ -451,7 -254,7 +455,8 @@@ page_dump(struct dump_request *dreq
      for (uint i=0; i<ep->pos; i++)
        RDUMP("    %p\n", ep->pages[i]);
    }
 +  UNLOCK_DOMAIN(resource, empty_pages_domain);
+   RDUMP("This request: %p\n", dreq);
  #endif
  }