From: Maria Matejka <mq@ucw.cz>
Date: Tue, 4 Feb 2025 20:13:26 +0000 (+0100)
Subject: Proper thread group setup
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c48d8172ca163d8973ef503d2a631e4c90ef9828;p=thirdparty%2Fbird.git

Proper thread group setup

Now the thread groups can be explicitly configured with their
latency and timing values.
---

diff --git a/conf/conf.h b/conf/conf.h
index 03ef4ef82..0367ebc46 100644
--- a/conf/conf.h
+++ b/conf/conf.h
@@ -67,7 +67,9 @@ struct config {
   char *err_file_name;			/* File name containing error */
   char *file_name;			/* Name of main configuration file */
   int file_fd;				/* File descriptor of main configuration file */
-  int thread_count;			/* How many worker threads to prefork */
+  int thread_group_simple;		/* Simple variant of thread configuration */
+  TLIST_LIST(thread_group_config) thread_group; /* Configured thread groups */
+  struct thread_group_config *default_thread_group;
 
   struct sym_scope *root_scope;		/* Scope for root symbols */
   struct sym_scope *current_scope;	/* Current scope where we are actually in while parsing */
@@ -95,6 +97,8 @@ struct global_runtime {
   u32 latency_limit;			/* Events with longer duration are logged (us) */
   u32 watchdog_warning;			/* I/O loop watchdog limit for warning (us) */
   u32 watchdog_timeout;			/* Watchdog timeout (in seconds, 0 = disabled) */
+
+  struct thread_group *default_thread_group;	/* Default thread group if not specified otherwise */
 };
 
 extern struct global_runtime * _Atomic global_runtime;
@@ -180,6 +184,7 @@ struct symbol {
     uint offset;			/* For SYM_VARIABLE */
     const struct keyword *keyword;	/* For SYM_KEYWORD */
     const struct f_method *method;	/* For SYM_METHOD */
+    struct thread_group_config *thread_group;	/* For SYM_THREAD_GROUP */
   };
 
   char name[0];
@@ -220,6 +225,7 @@ extern linpool *global_root_scope_linpool;
 #define SYM_METHOD 8
 #define SYM_MPLS_DOMAIN 9
 #define SYM_MPLS_RANGE 10
+#define SYM_THREAD_GROUP 11
 
 #define SYM_VARIABLE 0x100	/* 0x100-0x1ff are variable types */
 #define SYM_VARIABLE_RANGE SYM_VARIABLE ... (SYM_VARIABLE | 0xff)
diff --git a/doc/bird.sgml b/doc/bird.sgml
index a3563c666..f4d7247f7 100644
--- a/doc/bird.sgml
+++ b/doc/bird.sgml
@@ -607,12 +607,6 @@ include "tablename.conf";;
 	killed by abort signal. The timeout has effective granularity of
 	seconds, zero means disabled. Default: disabled (0).
 
-	<tag><label id="opt-threads">threads <m/number/</tag>
-	Set how many worker threads should BIRD spawn. Tests show that every
-        thread can utilize one complete CPU core, therefore you probably want to
-        keep at least one free core. The maximum feasible thread count heavily
-        depends on the actual workload and must be determined by testing or estimation. Default: 1
-
 	<tag><label id="opt-mrtdump">mrtdump "<m/filename/"</tag>
 	Set MRTdump file name. This option must be specified to allow MRTdump
 	feature. Default: no dump file.
@@ -722,6 +716,70 @@ include "tablename.conf";;
 	Evaluates given filter expression. It is used by the developers for testing of filters.
 </descrip>
 
+<sect1>Thread setup
+<label id="global-thread-setup">
+
+<p>BIRD runs in several threads. There is one main thread, taking care about startup,
+shutdown, (re)configuration, CLI and several protocols which have not yet been updated
+to run in other threads. Then there are several thread groups, running the rest.
+
+<p>Default thread group is <cf/worker/. This group runs (by default) BGP, BMP, MRT,
+Pipe and RPKI. Also the routing table maintenance routines run in these threads.
+For BFD, there is another thread group called
+<cf/express/, with minimal latency, expecting all tasks to be extremely fast.
+The threads are started as soon as some work is required from them, therefore
+if you don't configure BFD, the express thread won't start.
+
+<p>Any configuration of thread groups must be strictly placed before any table
+or protocol configuration, as these depend on the thread group existence.
+
+<p>Thread groups are configured by writing <cf>thread group <m/name/ {}</cf> blocks,
+which contain the following options:
+
+<p><descrip>
+	<tag><label id="thread-group-threads">threads <m/number/</tag>
+	Set how many threads should BIRD spawn in this thread group.
+	Every thread can utilize one complete CPU core. You probably want to
+        keep at least one free core for other processes. The maximum feasible
+	thread count heavily depends on the actual workload and must be determined
+	by testing or estimation. Default: 1
+
+	<tag><label id="thread-group-default">default <m/bool/</tag>
+	Mark this thread group as default. There must be exactly one thread group
+	marked as default. Default default thread group: <cf/worker/
+
+	<tag><label id="thread-max-latency">max latency <m/time/</tag>
+	Set maximum latency for the thread group. The group tries to dispatch every
+	event before this time elapses. Setting this too low causes BIRD
+	to spend lots of time on overhead and fill logs with reports.
+	Setting this too high causes BIRD to hold locks for long times.
+	Do not change unless you know what you are doing.
+	Default: 1 s for worker, 10 ms for express.
+
+	<tag><label id="thread-min-time">min time <m/time/</tag>
+	Set minimum awarded loop execution time for the thread group.
+	This is an internal performance tuning knob which may change between
+	minor versions. Do not change unless you know what you are doing.
+	Default: 10 ms for worker, 1 ms for express.
+
+	<tag><label id="thread-min-time">max time <m/time/</tag>
+	Set maximum awarded loop execution time for the thread group. This is
+	an internal performance tuning knob which may change between
+	minor versions. Do not change unless you know what you are doing.
+	Default: 300 ms for worker, 10 ms for express.
+
+	<tag><label id="thread-wakeup-time">wakeup time <m/time/</tag>
+	Set maximum sleep time for the thread group. Every thread will wake up
+	after this time even if no work is requested. This is an internal
+	IO loop knob which may change between minor versions.
+	Do not change unless you know what you are doing.
+	Default: 31415 s for worker, 60 s for express.
+</descrip>
+
+<p>There is also a "simple" thread group setting. If you write
+<cf>threads <m/number/</cf> on top level, it is equivalent to setting
+the worker group thread count to that number, and the express group thread count
+to one. This setting is deprecated and may disappear in some future version.
 
 <sect>Routing table options
 <label id="rtable-opts">
@@ -817,6 +875,9 @@ to set options.
 	debug. See also <ref id="channel-debug" name="channel debugging option">.
 	Default: off.
 
+	<tag><label id="proto-thread-group">thread group <m/name/</tag>
+	Assign this table's maintenance tasks to this thread group.
+	Default: <cf/worker/.
 </descrip>
 
 
@@ -901,6 +962,11 @@ agreement").
 	configuration for the list of supported channel names. See the
 	<ref id="channel-opts" name="channel configuration section"> for channel
 	definition.
+
+	<tag><label id="proto-thread-group">thread group <m/name/</tag>
+	Assign this protocol's tasks to this thread group. Ignored in protocols
+	not yet updated for the multithreaded execution model.
+	Default: <cf/worker/.
 </descrip>
 
 <p>There are several options that give sense only with certain protocols:
diff --git a/lib/io-loop.h b/lib/io-loop.h
index 9f08420a6..d2c27754c 100644
--- a/lib/io-loop.h
+++ b/lib/io-loop.h
@@ -34,7 +34,11 @@ bool task_before_halftime(void);
   } } while (0)
 
 /* Start a new birdloop owned by given pool and domain */
-struct birdloop *birdloop_new(pool *p, uint order, btime max_latency, const char *fmt, ...);
+typedef union thread_group_public thread_group;
+struct birdloop *birdloop_new(pool *p, uint order, thread_group *tg, const char *fmt, ...);
+
+/* Transfer the loop to a different thread group */
+void birdloop_transfer(struct birdloop *, thread_group *from, thread_group *to);
 
 /* Stop the loop. At the end, the @stopped callback is called unlocked in tail
  * position to finish cleanup. Run birdloop_free() from that callback to free
@@ -73,4 +77,28 @@ void birdloop_remove_socket(struct birdloop *, struct birdsock *);
 
 void birdloop_init(void);
 
+/* Configure threads */
+struct thread_group_config {
+#define TLIST_PREFIX thread_group_config
+#define TLIST_TYPE struct thread_group_config
+#define TLIST_ITEM n
+#define TLIST_WANT_ADD_TAIL
+#define TLIST_WANT_WALK
+  TLIST_DEFAULT_NODE;
+  thread_group *group;
+  struct symbol *symbol;
+  struct thread_params {
+    btime max_time;
+    btime min_time;
+    btime max_latency;
+    btime wakeup_time;
+  } params;
+  uint thread_count;
+};
+#include "lib/tlists.h"
+
+extern const struct thread_group_config thread_group_config_default_worker, thread_group_config_default_express;
+
+void thread_group_finalize_config(void);
+
 #endif /* _BIRD_IO_LOOP_H_ */
diff --git a/lib/locking.h b/lib/locking.h
index 60928ecc0..c44ea9e61 100644
--- a/lib/locking.h
+++ b/lib/locking.h
@@ -482,7 +482,6 @@ static inline void locking_unwind(struct lock_order *desired)
 #define LOBJ_PRIV(_obj, _level) \
   ({ ASSERT_DIE(DOMAIN_IS_LOCKED(_level, (_obj)->lock)); &(_obj)->priv; })
 
-
 /*
  * RCU retry unwinder
  *
diff --git a/nest/config.Y b/nest/config.Y
index 30ecc819b..b5e96a0b4 100644
--- a/nest/config.Y
+++ b/nest/config.Y
@@ -287,6 +287,11 @@ table_opt:
  | EXPORT SETTLE TIME settle { this_table->export_settle = $4; }
  | ROUTE REFRESH EXPORT SETTLE TIME settle { this_table->export_rr_settle = $6; }
  | DIGEST SETTLE TIME settle { this_table->digest_settle = $4; }
+ | THREAD GROUP symbol {
+    if ($3->class != SYM_THREAD_GROUP)
+      cf_error("Unexpected symbol %s as a thread group name", $3->name);
+    this_table->thread_group = $3->thread_group;
+ }
  ;
 
 table_opts:
@@ -350,6 +355,11 @@ proto_item:
  | DESCRIPTION text { this_proto->dsc = $2; }
  | VRF text { this_proto->vrf = if_get_by_name($2); }
  | VRF DEFAULT { this_proto->vrf = &default_vrf; }
+ | THREAD GROUP symbol {
+    if ($3->class != SYM_THREAD_GROUP)
+      cf_error("Unexpected symbol %s as a thread group name", $3->name);
+    this_proto->thread_group = $3->thread_group;
+ }
  ;
 
 
diff --git a/nest/proto.c b/nest/proto.c
index 64c41dcc5..99217dae5 100644
--- a/nest/proto.c
+++ b/nest/proto.c
@@ -1405,7 +1405,9 @@ proto_start(struct proto *p)
 
   if (p->cf->loop_order != DOMAIN_ORDER(the_bird))
   {
-    p->loop = birdloop_new(proto_pool, p->cf->loop_order, p->cf->loop_max_latency, "Protocol %s", p->cf->name);
+    p->loop = birdloop_new(proto_pool, p->cf->loop_order,
+	p->cf->thread_group ? p->cf->thread_group->group : NULL,
+	"Protocol %s", p->cf->name);
     p->pool = birdloop_pool(p->loop);
   }
   else
@@ -1458,6 +1460,9 @@ proto_config_new(struct protocol *pr, int class)
   cf->mrtdump = new_config->proto_default_mrtdump;
   cf->loop_order = DOMAIN_ORDER(the_bird);
 
+  thread_group_finalize_config();
+  cf->thread_group = new_config->default_thread_group;
+
   cf->restart_limit = 5 S;
 
   init_list(&cf->channels);
@@ -1591,6 +1596,9 @@ proto_reconfigure(struct proto *p, struct proto_config *oc, struct proto_config
   PD(p, "Reconfigured");
   p->cf = nc;
 
+  if (p->loop != &main_birdloop)
+    birdloop_transfer(p->loop, oc->thread_group->group, nc->thread_group->group);
+
   return 1;
 }
 
@@ -2198,7 +2206,7 @@ protos_build(void)
   struct settle_config cf = {.min = 0, .max = 0};
   proto_state_table_pub.journal.item_done = proto_journal_item_cleanup;
   proto_state_table_pub.journal.item_size = sizeof(struct proto_pending_update);
-  proto_state_table_pub.journal.loop = birdloop_new(&root_pool, DOMAIN_ORDER(service), 1, "proto journal loop");
+  proto_state_table_pub.journal.loop = birdloop_new(&root_pool, DOMAIN_ORDER(service), NULL, "proto journal loop");
   proto_state_table_pub.journal.domain = proto_state_table_pub.lock.rtable;
 
   lfjour_init(&proto_state_table_pub.journal, &cf);
diff --git a/nest/protocol.h b/nest/protocol.h
index ec561b263..9f91c94ce 100644
--- a/nest/protocol.h
+++ b/nest/protocol.h
@@ -114,7 +114,7 @@ struct proto_config {
   u32 debug, mrtdump;			/* Debugging bitfields, both use D_* constants */
   u32 router_id;			/* Protocol specific router ID */
   uint loop_order;			/* Launch a birdloop on this locking level; use DOMAIN_ORDER(the_bird) for mainloop */
-  btime loop_max_latency;		/* Request this specific maximum latency of loop; zero to default */
+  struct thread_group_config *thread_group;	/* Request this thread group for the birdloop */
   btime restart_limit;			/* Minimum allowed time between limit restarts */
 
   list channels;			/* List of channel configs (struct channel_config) */
diff --git a/nest/route.h b/nest/route.h
index fd46b0429..a994f7c5e 100644
--- a/nest/route.h
+++ b/nest/route.h
@@ -81,6 +81,7 @@ struct rtable_config {
     void (*setup)(union rtable *);
     void (*stop)(union rtable *);
   } master;				/* Data source (this table is aux) */
+  struct thread_group_config *thread_group;	/* Thread group to assign loops to */
 };
 
 /*
diff --git a/nest/rt-table.c b/nest/rt-table.c
index 78d344448..4fdb6d3a1 100644
--- a/nest/rt-table.c
+++ b/nest/rt-table.c
@@ -3354,7 +3354,7 @@ rt_setup(pool *pp, struct rtable_config *cf)
   ASSERT_DIE(birdloop_inside(&main_birdloop));
 
   /* Start the service thread */
-  struct birdloop *loop = birdloop_new(pp, DOMAIN_ORDER(service), 0, "Routing table service %s", cf->name);
+  struct birdloop *loop = birdloop_new(pp, DOMAIN_ORDER(service), cf->thread_group->group, "Routing table service %s", cf->name);
   birdloop_enter(loop);
   pool *sp = birdloop_pool(loop);
 
@@ -4596,6 +4596,9 @@ rt_new_table(struct symbol *s, uint addr_type)
       break;
   }
 
+  thread_group_finalize_config();
+  c->thread_group = new_config->default_thread_group;
+
   return c;
 }
 
@@ -4774,6 +4777,8 @@ rt_reconfigure(struct rtable_private *tab, struct rtable_config *new, struct rta
 
   tab->reconf_end = defer_call(&rrfdc.dc, sizeof rrfdc);
 
+  birdloop_transfer(tab->loop, old->thread_group->group, new->thread_group->group);
+
   return 1;
 }
 
@@ -4808,11 +4813,13 @@ rt_commit(struct config *new, struct config *old)
       WALK_LIST(o, old->tables)
       {
 	bool ok;
+	birdloop_enter(o->table->loop);
 	RT_LOCKED(o->table, tab)
 	{
 	  r = OBSREF_GET(tab->deleted) ? NULL : rt_find_table_config(new, o->name);
 	  ok = r && !new->shutdown && rt_reconfigure(tab, r, o);
 	}
+	birdloop_leave(o->table->loop);
 
 	if (ok)
 	  continue;
diff --git a/proto/bfd/config.Y b/proto/bfd/config.Y
index 56d1ffac4..9d336d665 100644
--- a/proto/bfd/config.Y
+++ b/proto/bfd/config.Y
@@ -39,7 +39,17 @@ bfd_proto_start: proto_start BFD
 {
   this_proto = proto_config_new(&proto_bfd, $1);
   this_proto->loop_order = DOMAIN_ORDER(proto);
-  this_proto->loop_max_latency = 10 MS_;
+  if (new_config->thread_group_simple)
+    this_proto->thread_group = TTAIL(thread_group, &new_config->thread_group);
+  else
+  {
+    struct symbol *sym = cf_find_symbol(new_config, "express");
+    if (sym && sym->class == SYM_THREAD_GROUP)
+      this_proto->thread_group = sym->thread_group;
+    else
+      this_proto->thread_group = NULL;
+  }
+
   init_list(&BFD_CFG->patt_list);
   init_list(&BFD_CFG->neigh_list);
   BFD_CFG->accept_ipv4 = BFD_CFG->accept_ipv6 = 1;
@@ -62,7 +72,10 @@ bfd_proto_opts:
  ;
 
 bfd_proto:
-   bfd_proto_start proto_name '{' bfd_proto_opts '}';
+   bfd_proto_start proto_name '{' bfd_proto_opts '}' {
+  if (!this_proto->thread_group)
+    cf_error("Thread group not configured and \"express\" not found.");
+} ;
 
 
 bfd_accept_item:
diff --git a/sysdep/unix/config.Y b/sysdep/unix/config.Y
index 42d78c6f6..b520a6d99 100644
--- a/sysdep/unix/config.Y
+++ b/sysdep/unix/config.Y
@@ -15,6 +15,7 @@ CF_DEFINES
 
 static struct log_config *this_log;
 static struct cli_config *this_cli_config;
+static struct thread_group_config *this_thread_group;
 
 CF_DECLS
 
@@ -22,6 +23,7 @@ CF_KEYWORDS(LOG, SYSLOG, ALL, DEBUG, TRACE, INFO, REMOTE, WARNING, ERROR, AUTH,
 CF_KEYWORDS(NAME, CONFIRM, UNDO, CHECK, TIMEOUT, DEBUG, LATENCY, LIMIT, WATCHDOG, WARNING, STATUS)
 CF_KEYWORDS(PING, WAKEUP, SOCKETS, SCHEDULING, EVENTS, TIMERS, ALLOCATOR)
 CF_KEYWORDS(GRACEFUL, RESTART, FIXED)
+CF_KEYWORDS(THREAD, THREADS, GROUP, MIN, MAX, TIME, LATENCY, DEFAULT)
 
 %type <i> log_mask log_mask_list log_cat cfg_timeout debug_unix latency_debug_mask latency_debug_flag latency_debug_list
 %type <t> cfg_name
@@ -145,10 +147,58 @@ cli_opts_block:
   cli_opts_block RESTRICT { this_cli_config->restricted = 1; }
 ;
 
-conf: THREADS expr {
-    if ($2 < 1) cf_error("Number of threads must be at least one.");
-    new_config->thread_count = $2;
-}
+conf: THREADS NUM {
+  if ($2 < 1) cf_error("Number of threads must be at least one.");
+  if (!new_config->thread_group_simple &&
+      !EMPTY_TLIST(thread_group_config, &new_config->thread_group))
+    cf_error("Mixing of `threads NUM` and `threads worker {}` is not allowed.");
+
+  if (new_config->thread_group_simple == -1)
+    THEAD(thread_group_config, &new_config->thread_group)->thread_count = $2;
+
+  new_config->thread_group_simple = $2;
+};
+
+conf: THREAD GROUP symbol {
+  switch (new_config->thread_group_simple) {
+    case -1: cf_error("Put `threads` block before any protocol or table definition.");
+    case 0: break;
+    default: cf_error("Mixing of `threads NUM` and `threads worker {}` is not allowed.");
+  }
+
+  this_thread_group = cfg_alloc(sizeof *this_thread_group);
+  *this_thread_group = strcmp($3->name, "express") ?
+    thread_group_config_default_worker :
+    thread_group_config_default_express;
+} '{' thread_group_opts '}' {
+  if (this_thread_group->params.max_time > 60 S_)
+    cf_error("Loop time maximum is 60 seconds, got %t", this_thread_group->params.max_time);
+  if (this_thread_group->params.max_latency < this_thread_group->params.max_time)
+    cf_error("Loop time maximum can't be higher than its latency limit");
+  if (this_thread_group->params.min_time > this_thread_group->params.max_time)
+    cf_error("Loop time minimm can't be higher than maximum");
+  if (this_thread_group->params.wakeup_time < 1 S_)
+    cf_error("Too low idle wakeup time, expected at least 1 second");
+
+  this_thread_group->symbol = cf_define_symbol(new_config, $3, SYM_THREAD_GROUP, thread_group, this_thread_group);
+  thread_group_config_add_tail(&new_config->thread_group, this_thread_group);
+  this_thread_group = NULL;
+};
+
+thread_group_opts: MAX TIME expr_us ';' { this_thread_group->params.max_time = $3; } ;
+thread_group_opts: MIN TIME expr_us ';' { this_thread_group->params.min_time = $3; } ;
+thread_group_opts: MAX LATENCY expr_us ';' { this_thread_group->params.max_latency = $3; } ;
+thread_group_opts: THREADS expr ';' { this_thread_group->thread_count = $2; } ;
+thread_group_opts: WAKEUP TIME expr_us ';' { this_thread_group->params.wakeup_time = $3; } ;
+thread_group_opts: DEFAULT bool ';' {
+  if ($2) {
+    if (new_config->default_thread_group &&
+	new_config->default_thread_group != this_thread_group)
+      cf_error("Too many default thread groups, already have %s",
+	  new_config->default_thread_group->symbol->name);
+    new_config->default_thread_group = this_thread_group;
+  }
+};
 
 
 conf: debug_unix ;
diff --git a/sysdep/unix/io-loop.c b/sysdep/unix/io-loop.c
index 8e60e402b..3be4a4ef6 100644
--- a/sysdep/unix/io-loop.c
+++ b/sysdep/unix/io-loop.c
@@ -568,59 +568,91 @@ sockets_fire(struct birdloop *loop, bool read, bool write)
  *	Threads
  */
 
-static void bird_thread_start_event(void *_data);
+#define TLIST_PREFIX thread_group
+#define TLIST_TYPE union thread_group_public
+#define TLIST_ITEM n
+#define TLIST_WANT_ADD_TAIL
+TLIST_DEFAULT_NODE;
+
+#define THREAD_GROUP_PUBLIC	\
+  DOMAIN(attrs) lock;		\
+  const char *name;		\
+  struct thread_group_node n;	\
 
 struct thread_group_private {
-  DOMAIN(attrs) lock;
+  THREAD_GROUP_PUBLIC;
+
   struct thread_group_private **locked_at;
   TLIST_LIST(birdloop) loops;
   TLIST_LIST(thread) threads;
+
   uint thread_count;
   uint thread_busy_count;
   uint loop_count;
   uint loop_unassigned_count;
-  btime max_latency;
-  event start_threads;
+  struct thread_params params;
   struct thread_dropper {
     struct birdloop *loop;
-    event *event;
-    uint goal;
+    event event;
+    OBSREF(struct config) conflock;
   } thread_dropper;
+  const struct thread_group_config *cf;
 };
 
 typedef union thread_group_public {
-  struct {
-    DOMAIN(attrs) lock;
-  };
+  struct { THREAD_GROUP_PUBLIC; };
   struct thread_group_private priv;
 } thread_group;
 
+#include "lib/tlists.h"
+
 #define TG_LOCKED(gpub, gpriv)	LOBJ_LOCKED(gpub, gpriv, thread_group, attrs)
 #define TG_LOCK(gpub, gpriv)	LOBJ_LOCK(gpub, gpriv, thread_group, attrs)
+#define TG_LOCKED_EXPR(gpub, gpriv, expr) ({ TG_LOCK(gpub, gpriv); expr; })
 
 LOBJ_UNLOCK_CLEANUP(thread_group, attrs);
 
-static thread_group pickup_groups[2] = {
-  {
-    /* all zeroes */
+static thread_group *default_thread_group;
+static TLIST_LIST(thread_group) global_thread_group_list;
+
+const struct thread_group_config
+thread_group_config_default_worker = {
+  .params = {
+    .max_time		= 300 MS_,
+    .min_time		= 10 MS_,
+    .max_latency	= 1 S_,
+    /* 8 hours, 43 minutes and 35 seconds to not conincide with anything */
+    .wakeup_time	= 31415 S_,
   },
-  { .priv = {
-    /* FIXME: make this dynamic, now it copies the loop_max_latency value from proto/bfd/config.Y */
-    .max_latency = 10 MS,
-    .start_threads.hook = bird_thread_start_event,
-    .start_threads.data = &pickup_groups[1],
-  }},
-};
+  .thread_count	= 1,
+},
+thread_group_config_default_express = {
+  .params = {
+    .max_time		= 10 MS_,
+    .min_time		= 1 MS_,
+    .max_latency	= 10 MS_,
+    .wakeup_time	= 60 S_,
+  },
+  .thread_count	= 1,
+},
+thread_group_shutdown = {};
+
 
 static _Thread_local struct bird_thread *this_thread;
 
 static void bird_thread_busy_set(struct thread_group_private *, int val);
 
 static void
-birdloop_set_thread(struct birdloop *loop, struct bird_thread *thr, thread_group *gpub)
+birdloop_set_thread(struct birdloop *loop, struct bird_thread *thr)
 {
   struct bird_thread *old = loop->thread;
-  ASSERT_DIE(!thr != !old);
+
+  /* We don't support direct reassignment from one thread to another */
+  ASSERT_DIE(!thr || !old);
+
+  /* We have to be in the thread we are trying to leave, for sure */
+  if (old)
+    ASSERT_DIE(birdloop_inside(old->meta));
 
   /* Signal our moving effort */
   u32 ltt = atomic_fetch_or_explicit(&loop->thread_transition, LTT_MOVE, memory_order_acq_rel);
@@ -635,7 +667,7 @@ birdloop_set_thread(struct birdloop *loop, struct bird_thread *thr, thread_group
   }
   /* Now we are free of running pings */
 
-  if (!thr)
+  if (old)
   {
     /* Unschedule from Meta */
     ev_postpone(&loop->event);
@@ -662,12 +694,12 @@ birdloop_set_thread(struct birdloop *loop, struct bird_thread *thr, thread_group
     ev_send_loop(loop->thread->meta, &loop->event);
   }
   else
-  {
-    /* Put into pickup list */
-    TG_LOCK(gpub, group);
-    birdloop_add_tail(&group->loops, loop);
-    group->loop_unassigned_count++;
-  }
+    TG_LOCKED(loop->thread_group, group)
+    {
+      /* Put into pickup list */
+      birdloop_add_tail(&group->loops, loop);
+      group->loop_unassigned_count++;
+    }
 
   loop->last_transition_ns = ns_now();
 }
@@ -734,6 +766,10 @@ birdloop_balancer(void)
 
   TG_LOCKED(this_thread->group, group)
   {
+    /* Update timing parameters */
+    this_thread->params = group->params;
+
+    /* Check drop */
     drop_needed =
       this_thread->busy_active &&
       (group->thread_busy_count < group->thread_count) &&
@@ -769,7 +805,7 @@ birdloop_balancer(void)
 	LOOP_TRACE(loop, DL_SCHEDULING, "Dropping from thread, remaining %u loops here", this_thread->loop_count);
 
 	/* This also unschedules the loop from Meta */
-	birdloop_set_thread(loop, NULL, this_thread->group);
+	birdloop_set_thread(loop, NULL);
 
 	dropped++;
 	if ((dropped * dropped) / 2 > this_thread->loop_count)
@@ -809,12 +845,19 @@ birdloop_balancer(void)
     struct birdloop *loop = pick_this;
 
     birdloop_enter(loop);
-    birdloop_set_thread(loop, this_thread, this_thread->group);
-    LOOP_TRACE(loop, DL_SCHEDULING, "Picked up by thread");
 
-    node *n;
-    WALK_LIST(n, loop->sock_list)
-      SKIP_BACK(sock, n, n)->index = -1;
+    if (loop->thread_group == this_thread->group)
+    {
+      birdloop_set_thread(loop, this_thread);
+      LOOP_TRACE(loop, DL_SCHEDULING, "Picked up by thread");
+
+      node *n;
+      WALK_LIST(n, loop->sock_list)
+	SKIP_BACK(sock, n, n)->index = -1;
+    }
+    else
+      /* Transfer required, drop immediately */
+      birdloop_set_thread(loop, NULL);
 
     birdloop_leave(loop);
 
@@ -897,9 +940,12 @@ bird_thread_main(void *arg)
     u64 thr_before_run = ns_now();
     if (thr->loop_count > 0)
     {
-      thr->max_loop_time_ns = (thr->max_latency_ns / 2 - (thr_before_run - thr_loop_start)) / (u64) thr->loop_count;
-      if (thr->max_loop_time_ns NS > 300 MS)
-	thr->max_loop_time_ns = 300 MS TO_NS;
+      thr->max_loop_time_ns = ((thr->params.max_latency TO_NS) / 2 - (thr_before_run - thr_loop_start)) / (u64) thr->loop_count;
+      if (thr->max_loop_time_ns NS > (u64) thr->params.max_time)
+	thr->max_loop_time_ns = thr->params.max_time TO_NS;
+      if (thr->max_loop_time_ns NS < (u64) thr->params.min_time)
+	thr->max_loop_time_ns = thr->params.min_time TO_NS;
+	/* TODO: put a warning about possible overload here */
     }
 
     /* Run all scheduled loops */
@@ -1000,11 +1046,41 @@ poll_retry:;
   bug("An infinite loop has ended.");
 }
 
+static void
+bird_thread_group_done(thread_group *gpub, TLIST_LIST(birdloop) *leftover_loops)
+{
+  /* Sanity checks */
+  ASSERT_DIE(birdloop_inside(&main_birdloop));
+  TG_LOCKED(gpub, group)
+    ASSERT_DIE(EMPTY_TLIST(thread, &group->threads));
+
+  /* Transfer loops left here to the default group */
+  while (!EMPTY_TLIST(birdloop, leftover_loops))
+  {
+    struct birdloop *loop = THEAD(birdloop, leftover_loops);
+    birdloop_enter(loop);
+    if (loop->thread_group == gpub)
+    {
+      birdloop_transfer(loop, gpub, default_thread_group);
+    }
+
+    birdloop_rem_node(leftover_loops, loop);
+    birdloop_set_thread(loop, NULL);
+    birdloop_leave(loop);
+  }
+
+  thread_group_rem_node(&global_thread_group_list, gpub);
+  DOMAIN_FREE(attrs, gpub->lock);
+  mb_free(gpub);
+}
+
 static void
 bird_thread_cleanup(void *_thr)
 {
   struct bird_thread *thr = _thr;
   struct birdloop *meta = thr->meta;
+  thread_group *gpub = thr->group;
+
   ASSERT_DIE(birdloop_inside(&main_birdloop));
 
   /* Wait until the thread actually finishes */
@@ -1022,63 +1098,99 @@ bird_thread_cleanup(void *_thr)
   thr->meta->thread = NULL;
   thr->meta = NULL;
   birdloop_free(meta);
+
+  TLIST_LIST(birdloop) *leftover_loops = NULL;
+  struct birdloop *tdf = NULL;
+
+  TG_LOCKED(gpub, group) {
+    if ((group->cf == &thread_group_shutdown) && EMPTY_TLIST(thread, &group->threads))
+    {
+      OBSREF_CLEAR(group->thread_dropper.conflock);
+      tdf = group->thread_dropper.loop;
+      birdloop_rem_node(&group->loops, tdf);
+      group->thread_dropper.loop = NULL;
+      group->thread_dropper.event = (event) {};
+      leftover_loops = &group->loops;
+    }
+  }
+
+  if (tdf)
+    birdloop_free(tdf);
+
+  if (leftover_loops)
+    /* Happens only with the last thread */
+    bird_thread_group_done(gpub, leftover_loops);
 }
 
-static struct bird_thread *
-bird_thread_start(thread_group *gpub)
+static void bird_thread_start(thread_group *);
+static void
+bird_thread_start_event(void *_data)
 {
-  ASSERT_DIE(birdloop_inside(&main_birdloop));
+  thread_group *group = _data;
+  bird_thread_start(group);
+}
 
-  struct birdloop *meta = birdloop_new_no_pickup(&root_pool, DOMAIN_ORDER(meta), "Thread Meta");
-  pool *p = birdloop_pool(meta);
+static void
+bird_thread_start_indirect(struct thread_group_private *group)
+{
+  group->thread_dropper.event.hook = bird_thread_start_event;
+  ev_send(&global_event_list, &group->thread_dropper.event);
+}
 
-  birdloop_enter(meta);
-  struct bird_thread *thr = mb_allocz(p, sizeof(*thr));
-  thr->pool = p;
+static void
+bird_thread_start(thread_group *gpub)
+{
+  ASSERT_DIE(birdloop_inside(&main_birdloop));
 
-  TG_LOCKED(gpub, group)
+  while (true)
   {
+    /* Already enough threads */
+    TG_LOCKED(gpub, group)
+      if (group->thread_count >= group->cf->thread_count)
+	return;
 
-  thr->cleanup_event = (event) { .hook = bird_thread_cleanup, .data = thr, };
-  thr->group = gpub;
-  thr->max_latency_ns = (group->max_latency ?: 5 S) TO_NS;
-  thr->meta = meta;
-  thr->meta->thread = thr;
+    struct birdloop *meta = birdloop_new_no_pickup(&root_pool, DOMAIN_ORDER(meta), "Thread Meta");
+    pool *p = birdloop_pool(meta);
 
-  wakeup_init(thr);
-  ev_init_list(&thr->priority_events, NULL, "Thread direct event list");
+    birdloop_enter(meta);
+    struct bird_thread *thr = mb_allocz(p, sizeof(*thr));
+    thr->pool = p;
 
-  thread_add_tail(&group->threads, thr);
+    TG_LOCKED(gpub, group)
+    {
+      thr->cleanup_event = (event) { .hook = bird_thread_cleanup, .data = thr, };
+      thr->group = gpub;
+      thr->params = group->params;
+      thr->meta = meta;
+      thr->meta->thread = thr;
 
-  int e = 0;
+      wakeup_init(thr);
+      ev_init_list(&thr->priority_events, NULL, "Thread direct event list");
 
-  if (e = pthread_attr_init(&thr->thread_attr))
-    die("pthread_attr_init() failed: %M", e);
+      thread_add_tail(&group->threads, thr);
 
-  /* We don't have to worry about thread stack size so much.
-  if (e = pthread_attr_setstacksize(&thr->thread_attr, THREAD_STACK_SIZE))
-    die("pthread_attr_setstacksize(%u) failed: %M", THREAD_STACK_SIZE, e);
-    */
+      int e = 0;
 
-  if (e = pthread_attr_setdetachstate(&thr->thread_attr, PTHREAD_CREATE_DETACHED))
-    die("pthread_attr_setdetachstate(PTHREAD_CREATE_DETACHED) failed: %M", e);
+      if (e = pthread_attr_init(&thr->thread_attr))
+	die("pthread_attr_init() failed: %M", e);
 
-  if (e = pthread_create(&thr->thread_id, &thr->thread_attr, bird_thread_main, thr))
-    die("pthread_create() failed: %M", e);
+      /* We don't have to worry about thread stack size so much.
+	 if (e = pthread_attr_setstacksize(&thr->thread_attr, THREAD_STACK_SIZE))
+	 die("pthread_attr_setstacksize(%u) failed: %M", THREAD_STACK_SIZE, e);
+	 */
 
-  group->thread_count++;
+      if (e = pthread_attr_setdetachstate(&thr->thread_attr, PTHREAD_CREATE_DETACHED))
+	die("pthread_attr_setdetachstate(PTHREAD_CREATE_DETACHED) failed: %M", e);
 
-  }
+      if (e = pthread_create(&thr->thread_id, &thr->thread_attr, bird_thread_main, thr))
+	die("pthread_create() failed: %M", e);
 
-  birdloop_leave(meta);
-  return thr;
-}
+      group->thread_count++;
 
-static void
-bird_thread_start_event(void *_data)
-{
-  thread_group *group = _data;
-  bird_thread_start(group);
+    }
+
+    birdloop_leave(meta);
+  }
 }
 
 static void
@@ -1096,14 +1208,16 @@ bird_thread_shutdown(void * _ UNUSED)
 
   TG_LOCKED(this_thread->group, group)
   {
-    dif = group->thread_count - group->thread_dropper.goal;
+    dif = group->thread_count - group->cf->thread_count;
 
     if (dif > 0)
-      ev_send_loop(group->thread_dropper.loop, group->thread_dropper.event);
+      ev_send_loop(group->thread_dropper.loop, &group->thread_dropper.event);
     else
     {
+      ASSERT_DIE(dif == 0);
       tdl_stop = group->thread_dropper.loop;
       group->thread_dropper.loop = NULL;
+      OBSREF_CLEAR(group->thread_dropper.conflock);
     }
   }
 
@@ -1147,12 +1261,13 @@ bird_thread_shutdown(void * _ UNUSED)
     birdloop_rem_node(&thr->loops, loop);
 
     /* Unset loop's thread */
-    birdloop_set_thread(loop, NULL, thr->group);
+    birdloop_set_thread(loop, NULL);
   }
 
   /* Let others know about new loops */
   TG_LOCKED(this_thread->group, group)
-    if (!EMPTY_TLIST(birdloop, &group->loops))
+    if (!EMPTY_TLIST(birdloop, &group->loops)
+	&& !EMPTY_TLIST(thread, &group->threads))
       wakeup_do_kick(THEAD(thread, &group->threads));
 
   /* Request thread cleanup from main loop */
@@ -1172,55 +1287,167 @@ bird_thread_shutdown(void * _ UNUSED)
   pthread_exit(NULL);
 }
 
+static void
+bird_thread_stop(thread_group *gpub, struct config *old_config)
+{
+  struct birdloop *tdl = birdloop_new(&root_pool, DOMAIN_ORDER(control), gpub, "Thread dropper");
+  birdloop_enter(tdl);
+
+  TG_LOCKED(gpub, group)
+  {
+    group->thread_dropper = (struct thread_dropper) {
+      .loop = tdl,
+      .event.hook = bird_thread_shutdown,
+    };
+    OBSREF_SET(group->thread_dropper.conflock, old_config);
+    ev_send_loop(tdl, &group->thread_dropper.event);
+  }
+
+  birdloop_leave(tdl);
+}
+
 void
-bird_thread_commit(struct config *new, struct config *old UNUSED)
+bird_thread_commit(struct config *new, struct config *old)
 {
   ASSERT_DIE(birdloop_inside(&main_birdloop));
 
   if (new->shutdown)
     return;
 
-  if (!new->thread_count)
-    new->thread_count = 1;
-
-  while (1)
+  /* First, we match the new config to the existing groups */
+  WALK_TLIST(thread_group_config, tgc, &new->thread_group)
   {
-    int dif;
-    bool thread_dropper_running;
+    thread_group *found = NULL;
+    int dif = -tgc->thread_count;
+    bool thread_dropper_running = false;
 
-    TG_LOCKED(&pickup_groups[0], group)
+    WALK_TLIST(thread_group, gpub, &global_thread_group_list)
     {
-      dif = group->thread_count - new->thread_count;
-      thread_dropper_running = !!group->thread_dropper.loop;
+      TG_LOCKED(gpub, group)
+	if (!strcmp(group->name, tgc->symbol->name))
+	{
+	  ASSERT_DIE(thread_group_config_enlisted(group->cf) == &old->thread_group);
+	  found = gpub;
+	  group->cf = tgc;
+	  tgc->group = gpub;
+	  group->name = tgc->symbol->name;
+	  /* Do not start threads for empty groups */
+	  if ((group->thread_count == 0) && EMPTY_TLIST(birdloop, &group->loops))
+	    dif = 0;
+	  else
+	    dif = group->thread_count - tgc->thread_count;
+	  thread_dropper_running = !!group->thread_dropper.loop;
+	  group->params = tgc->params;
+	}
+
+      if (found)
+	break;
     }
 
-    if (dif < 0)
+    if (found)
     {
-      bird_thread_start(&pickup_groups[0]);
-      continue;
+      if (dif < 0)
+	bird_thread_start(found);
+      else if ((dif > 0) && !thread_dropper_running)
+	bird_thread_stop(found, old);
     }
+    else
+    {
+      struct thread_group_private *group = mb_allocz(&root_pool, sizeof *group);
+      SKIP_BACK_DECLARE(thread_group, gpub, priv, group);
+      group->lock = DOMAIN_NEW(attrs);
+      DOMAIN_SETUP(attrs, group->lock, "Thread Group", NULL);
+      group->cf = tgc;
+      tgc->group = gpub;
+      group->params = tgc->params;
+      group->name = tgc->symbol->name;
+      group->thread_dropper.event = (event) { .data = group, };
+
+      thread_group_add_tail(&global_thread_group_list, gpub);
+      /* Will start threads when some loop emerges, not now. */
+    }
+  }
+
+  ASSERT_DIE(new->default_thread_group);
+  default_thread_group = new->default_thread_group->group;
 
-    if ((dif > 0) && !thread_dropper_running)
+  WALK_TLIST(thread_group, gpub, &global_thread_group_list)
+  {
+    bool run_thread_dropper = false;
+    TLIST_LIST(birdloop) *leftover_loops = NULL;
+
+    TG_LOCKED(gpub, group)
     {
-      struct birdloop *tdl = birdloop_new(&root_pool, DOMAIN_ORDER(control), pickup_groups[0].priv.max_latency, "Thread dropper");
-      birdloop_enter(tdl);
-      event *tde = ev_new_init(tdl->pool, bird_thread_shutdown, NULL);
+      if (group->cf && (thread_group_config_enlisted(group->cf) == &new->thread_group))
+	break; /* Unlock, group already reconfigured */
 
-      TG_LOCKED(&pickup_groups[0], group)
-      {
-	group->thread_dropper = (struct thread_dropper) {
-	  .loop = tdl,
-	  .event = tde,
-	  .goal = new->thread_count,
-	};
-      }
+      /* All shutting-down groups are expected to be finished
+       * before another config commit */
+      ASSERT_DIE(group->cf != &thread_group_shutdown);
+
+      /* The thread dropper should not be running now,
+       * it blocks config completion */
+      ASSERT_DIE(!group->thread_dropper.loop);
+
+      /* The only case the thread_dropper event runs,
+       * is when it was recently summonned to start some new threads */
+      ev_postpone(&group->thread_dropper.event);
 
-      ev_send_loop(tdl, tde);
-      birdloop_leave(tdl);
+      /* Needa stop the group */
+      ASSERT_DIE(!group->cf || (thread_group_config_enlisted(group->cf) == &old->thread_group));
+      group->cf = &thread_group_shutdown;
+
+      if (EMPTY_TLIST(thread, &group->threads))
+	leftover_loops = &group->loops;
+      else
+	run_thread_dropper = true;
     }
 
-    return;
+    /* Drop loops immediately, no thread to kill */
+    if (leftover_loops)
+      bird_thread_group_done(gpub, leftover_loops);
+
+    /* Kill threads, loops get dropped later */
+    if (run_thread_dropper)
+      bird_thread_stop(gpub, old);
   }
+
+  /* after bird_thread_stop(), the old config reference is blocked
+   * until the threads have finished thread stopping. */
+}
+
+void
+thread_group_finalize_config(void)
+{
+  if (EMPTY_TLIST(thread_group_config, &new_config->thread_group))
+  {
+    if (!new_config->thread_group_simple)
+      new_config->thread_group_simple = -1;
+
+    /* Default worker thread group */
+    struct thread_group_config *tgc = cfg_alloc(sizeof *tgc);
+    *tgc = thread_group_config_default_worker;
+    tgc->thread_count = (new_config->thread_group_simple > 0) ?
+      new_config->thread_group_simple : 1;
+    thread_group_config_add_tail(&new_config->thread_group, tgc);
+    new_config->default_thread_group = tgc;
+
+    tgc->symbol = cf_define_symbol(
+	new_config, cf_get_symbol(new_config, "worker"),
+	SYM_THREAD_GROUP, thread_group, tgc);
+
+    /* Default express thread group */
+    tgc = cfg_alloc(sizeof *tgc);
+    *tgc = thread_group_config_default_express;
+    thread_group_config_add_tail(&new_config->thread_group, tgc);
+
+    tgc->symbol = cf_define_symbol(
+	new_config, cf_get_symbol(new_config, "express"),
+	SYM_THREAD_GROUP, thread_group, tgc);
+  }
+
+  if (!new_config->default_thread_group)
+    cf_error("No default thread group configured");
 }
 
 /* Cleanup after last thread */
@@ -1263,6 +1490,8 @@ bird_thread_sync_all(struct bird_thread_syncer *sync,
     void (*hook)(struct bird_thread_syncer *),
     void (*done)(struct bird_thread_syncer *), const char *name)
 {
+  ASSERT_DIE(birdloop_inside(&main_birdloop));
+
   sync->lock = DOMAIN_NEW(control);
   LOCK_DOMAIN(control, sync->lock);
 
@@ -1270,8 +1499,8 @@ bird_thread_sync_all(struct bird_thread_syncer *sync,
   sync->hook = hook;
   sync->finish = done;
 
-  for (int i=0; i<2; i++)
-    TG_LOCKED(&pickup_groups[i], group)
+  WALK_TLIST(thread_group, gpub, &global_thread_group_list)
+    TG_LOCKED(gpub, group)
       WALK_TLIST(thread, thr, &group->threads)
       {
 	sync->total++;
@@ -1388,14 +1617,15 @@ cmd_show_threads_done(struct bird_thread_syncer *sync)
   tsd->cli->cont = NULL;
   tsd->cli->cleanup = NULL;
 
-  for (int i=0; i<2; i++) TG_LOCKED(&pickup_groups[i], group)
+  WALK_TLIST(thread_group, gpub, &global_thread_group_list)
+  TG_LOCKED(gpub, group)
   {
     uint count = 0;
     u64 total_time_ns = 0;
     if (!EMPTY_TLIST(birdloop, &group->loops))
     {
       if (tsd->show_loops)
-	tsd_append("Unassigned loops in group %d:", i);
+	tsd_append("Unassigned loops in group %s:", group->name);
 
       WALK_TLIST(birdloop, loop, &group->loops)
       {
@@ -1409,10 +1639,10 @@ cmd_show_threads_done(struct bird_thread_syncer *sync)
       if (tsd->show_loops)
 	tsd_append("  Total working time: %t", total_time_ns NS);
       else
-	tsd_append("Unassigned %d loops in group %d, total time %t", count, i, total_time_ns NS);
+	tsd_append("Unassigned %d loops in group %s, total time %t", count, group->name, total_time_ns NS);
     }
     else
-      tsd_append("All loops in group %d are assigned.", i);
+      tsd_append("All loops in group %s are assigned.", group->name);
   }
 
   if (!tsd->show_loops)
@@ -1471,13 +1701,18 @@ birdloop_init(void)
 {
   ns_init();
 
-  for (int i=0; i<2; i++)
-  {
-    struct thread_group_private *group = &pickup_groups[i].priv;
-
-    group->lock = DOMAIN_NEW(attrs);
-    DOMAIN_SETUP(attrs, group->lock, "Loop Pickup", NULL);
-  }
+  default_thread_group = mb_allocz(&root_pool, sizeof *default_thread_group);
+  *default_thread_group = (thread_group) {
+    .priv = {
+      .name = "startup",
+      .lock = DOMAIN_NEW(attrs),
+      .thread_dropper.event = {
+	.data = default_thread_group,
+      },
+    },
+  };
+  DOMAIN_SETUP(attrs, default_thread_group->priv.lock, "Startup Thread Group", NULL);
+  thread_group_add_tail(&global_thread_group_list, default_thread_group);
 
   wakeup_init(main_birdloop.thread);
 
@@ -1568,6 +1803,13 @@ birdloop_run(void *_loop)
     birdloop_yield();
   }
 
+  if (loop->thread_group != this_thread->group)
+  {
+    birdloop_rem_node(&this_thread->loops, loop);
+    birdloop_set_thread(loop, NULL);
+    goto leave;
+  }
+
   /* Now we can actually do some work */
   u64 dif = account_to(&loop->working);
 
@@ -1618,6 +1860,7 @@ birdloop_run(void *_loop)
   this_thread->sock_changed |= loop->sock_changed;
   loop->sock_changed = 0;
 
+leave:
   account_to(&this_thread->overhead);
   this_birdloop = this_thread->meta;
   birdloop_leave(loop);
@@ -1644,6 +1887,8 @@ birdloop_vnew_internal(pool *pp, uint order, thread_group *gpub, const char *nam
   loop->time.domain = dg;
   loop->time.loop = loop;
 
+  loop->thread_group = gpub;
+
   atomic_store_explicit(&loop->thread_transition, 0, memory_order_relaxed);
 
   birdloop_enter_locked(loop);
@@ -1658,14 +1903,17 @@ birdloop_vnew_internal(pool *pp, uint order, thread_group *gpub, const char *nam
   LOOP_TRACE(loop, DL_SCHEDULING, "New loop: %s", p->name);
 
   if (gpub)
+    /* Send the loop to the requested thread group for execution */
     TG_LOCKED(gpub, group)
     {
       group->loop_count++;
       group->loop_unassigned_count++;
       birdloop_add_tail(&group->loops, loop);
       if (EMPTY_TLIST(thread, &group->threads))
-	ev_send(&global_event_list, &group->start_threads);
+	/* If no threads are there for the loop, request them to start */
+	bird_thread_start_indirect(group);
       else
+	/* Just wakeup the first one thread to pick us up */
 	wakeup_do_kick(THEAD(thread, &group->threads));
     }
 
@@ -1685,15 +1933,51 @@ birdloop_new_no_pickup(pool *pp, uint order, const char *name, ...)
 }
 
 struct birdloop *
-birdloop_new(pool *pp, uint order, btime max_latency, const char *name, ...)
+birdloop_new(pool *pp, uint order, thread_group *group, const char *name, ...)
 {
+  if (!group)
+    group = default_thread_group;
+
   va_list args;
   va_start(args, name);
-  struct birdloop *loop = birdloop_vnew_internal(pp, order, max_latency ? &pickup_groups[1] : &pickup_groups[0], name, args);
+  struct birdloop *loop = birdloop_vnew_internal(pp, order, group, name, args);
   va_end(args);
   return loop;
 }
 
+static void
+birdloop_transfer_dummy_event(void *_data)
+{
+  rfree(_data);
+}
+
+void
+birdloop_transfer(struct birdloop *loop, thread_group *from, thread_group *to)
+{
+  ASSERT_DIE(birdloop_inside(loop));
+  if (loop->thread_group != from)
+  {
+    log(L_WARN "Failed to transfer loop %s from group %s to %s, now in %s",
+	LOOP_NAME(loop), from->name, to->name, loop->thread->group->name);
+    birdloop_leave(loop);
+    return;
+  }
+
+  /* Set the new group, actually */
+  loop->thread_group = to;
+
+  /* Request the loop to actually do something to get scheduled */
+  event *e = ev_new(loop->pool);
+  e->hook = birdloop_transfer_dummy_event;
+  e->data = e;
+  ev_send_loop(loop, e);
+
+  /* Possibly start threads in that group */
+  TG_LOCKED(to, group)
+    if (EMPTY_TLIST(thread, &group->threads))
+      bird_thread_start_indirect(group);
+}
+
 static void
 birdloop_do_stop(struct birdloop *loop, void (*stopped)(void *data), void *data)
 {
diff --git a/sysdep/unix/io-loop.h b/sysdep/unix/io-loop.h
index 1b8aaf9b7..6681f4d6d 100644
--- a/sysdep/unix/io-loop.h
+++ b/sysdep/unix/io-loop.h
@@ -73,6 +73,7 @@ struct birdloop
   struct birdloop *prev_loop;
 
   struct bird_thread *thread;
+  thread_group *thread_group;
 
 #define TIME_BY_SEC_SIZE	16
   struct spent_time working, locking;
@@ -110,7 +111,7 @@ struct bird_thread
   u16 busy_counter;
   uint loop_count;
 
-  u64 max_latency_ns;
+  struct thread_params params;
   u64 max_loop_time_ns;
 
   struct spent_time overhead, idle;
diff --git a/sysdep/unix/unix.h b/sysdep/unix/unix.h
index c4130edf6..e936e5e28 100644
--- a/sysdep/unix/unix.h
+++ b/sysdep/unix/unix.h
@@ -41,6 +41,7 @@ void cmd_graceful_restart(void);
 void cmd_show_threads(int);
 void bird_thread_commit(struct config *new, struct config *old);
 
+
 #define UNIX_DEFAULT_CONFIGURE_TIMEOUT	300
 
 #define UNIX_DEFAULT_LATENCY_LIMIT	(1 S_)