rbtree: Provide rbtree with links

Some RB tree users require quick access to the next and the previous node,
e.g. to check whether a modification of the node results in a change of the
nodes position in the tree. If the node position does not change, then the
modification can happen in place without going through a full enqueue
requeue cycle. A upcoming use case for this are the timer queues of the
hrtimer subsystem as they can optimize for timers which are frequently
rearmed while enqueued.

This can be obviously achieved with rb_next() and rb_prev(), but those
turned out to be quite expensive for hotpath operations depending on the
tree depth.

Add a linked RB tree variant where add() and erase() maintain the links
between the nodes. Like the cached variant it provides a pointer to the
left most node in the root.

It intentionally does not use a [h]list head as there is no real need for
true list operations as the list is strictly coupled to the tree and
and cannot be manipulated independently.

It sets the nodes previous pointer to NULL for the left most node and the
next pointer to NULL for the right most node. This allows a quick check
especially for the left most node without consulting the list head address,
which creates better code.

Aside of the rb_leftmost cached pointer this could trivially provide a
rb_rightmost pointer as well, but there is no usage for that (yet).

Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260224163431.668401024@kernel.org

diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h
index 4091e978aef2404b56d7643d9385727c69796678..48acdc3889dd4e76a2313d822407a570c1b0fe90 100644 (file)
--- a/include/linux/rbtree.h
+++ b/include/linux/rbtree.h
@@ -35,10 +35,15 @@
 #define RB_CLEAR_NODE(node)  \
        ((node)->__rb_parent_color = (unsigned long)(node))
 
+#define RB_EMPTY_LINKED_NODE(lnode)  RB_EMPTY_NODE(&(lnode)->node)
+#define RB_CLEAR_LINKED_NODE(lnode)  ({                                        \
+       RB_CLEAR_NODE(&(lnode)->node);                                  \
+       (lnode)->prev = (lnode)->next = NULL;                           \
+})
 
 extern void rb_insert_color(struct rb_node *, struct rb_root *);
 extern void rb_erase(struct rb_node *, struct rb_root *);
-
+extern bool rb_erase_linked(struct rb_node_linked *, struct rb_root_linked *);
 
 /* Find logical next and previous nodes in a tree */
 extern struct rb_node *rb_next(const struct rb_node *);
@@ -213,15 +218,10 @@ rb_add_cached(struct rb_node *node, struct rb_root_cached *tree,
        return leftmost ? node : NULL;
 }
 
-/**
- * rb_add() - insert @node into @tree
- * @node: node to insert
- * @tree: tree to insert @node into
- * @less: operator defining the (partial) node order
- */
 static __always_inline void
-rb_add(struct rb_node *node, struct rb_root *tree,
-       bool (*less)(struct rb_node *, const struct rb_node *))
+__rb_add(struct rb_node *node, struct rb_root *tree,
+        bool (*less)(struct rb_node *, const struct rb_node *),
+        void (*linkop)(struct rb_node *, struct rb_node *, struct rb_node **))
 {
        struct rb_node **link = &tree->rb_node;
        struct rb_node *parent = NULL;
@@ -234,10 +234,73 @@ rb_add(struct rb_node *node, struct rb_root *tree,
                        link = &parent->rb_right;
        }
 
+       linkop(node, parent, link);
        rb_link_node(node, parent, link);
        rb_insert_color(node, tree);
 }
 
+#define __node_2_linked_node(_n) \
+       rb_entry((_n), struct rb_node_linked, node)
+
+static inline void
+rb_link_linked_node(struct rb_node *node, struct rb_node *parent, struct rb_node **link)
+{
+       if (!parent)
+               return;
+
+       struct rb_node_linked *nnew = __node_2_linked_node(node);
+       struct rb_node_linked *npar = __node_2_linked_node(parent);
+
+       if (link == &parent->rb_left) {
+               nnew->prev = npar->prev;
+               nnew->next = npar;
+               npar->prev = nnew;
+               if (nnew->prev)
+                       nnew->prev->next = nnew;
+       } else {
+               nnew->next = npar->next;
+               nnew->prev = npar;
+               npar->next = nnew;
+               if (nnew->next)
+                       nnew->next->prev = nnew;
+       }
+}
+
+/**
+ * rb_add_linked() - insert @node into the leftmost linked tree @tree
+ * @node: node to insert
+ * @tree: linked tree to insert @node into
+ * @less: operator defining the (partial) node order
+ *
+ * Returns @true when @node is the new leftmost, @false otherwise.
+ */
+static __always_inline bool
+rb_add_linked(struct rb_node_linked *node, struct rb_root_linked *tree,
+             bool (*less)(struct rb_node *, const struct rb_node *))
+{
+       __rb_add(&node->node, &tree->rb_root, less, rb_link_linked_node);
+       if (!node->prev)
+               tree->rb_leftmost = node;
+       return !node->prev;
+}
+
+/* Empty linkop function which is optimized away by the compiler */
+static __always_inline void
+rb_link_noop(struct rb_node *n, struct rb_node *p, struct rb_node **l) { }
+
+/**
+ * rb_add() - insert @node into @tree
+ * @node: node to insert
+ * @tree: tree to insert @node into
+ * @less: operator defining the (partial) node order
+ */
+static __always_inline void
+rb_add(struct rb_node *node, struct rb_root *tree,
+       bool (*less)(struct rb_node *, const struct rb_node *))
+{
+       __rb_add(node, tree, less, rb_link_noop);
+}
+
 /**
  * rb_find_add_cached() - find equivalent @node in @tree, or add @node
  * @node: node to look-for / insert

diff --git a/include/linux/rbtree_types.h b/include/linux/rbtree_types.h
index 45b6ecde3665aa744f790cd915445fe07595181c..3c7ae53e813976bd340e3c91c8833910ed1fda56 100644 (file)
--- a/include/linux/rbtree_types.h
+++ b/include/linux/rbtree_types.h
@@ -9,6 +9,12 @@ struct rb_node {
 } __attribute__((aligned(sizeof(long))));
 /* The alignment might seem pointless, but allegedly CRIS needs it */
 
+struct rb_node_linked {
+       struct rb_node          node;
+       struct rb_node_linked   *prev;
+       struct rb_node_linked   *next;
+};
+
 struct rb_root {
        struct rb_node *rb_node;
 };
@@ -28,7 +34,17 @@ struct rb_root_cached {
        struct rb_node *rb_leftmost;
 };
 
+/*
+ * Leftmost tree with links. This would allow a trivial rb_rightmost update,
+ * but that has been omitted due to the lack of users.
+ */
+struct rb_root_linked {
+       struct rb_root          rb_root;
+       struct rb_node_linked   *rb_leftmost;
+};
+
 #define RB_ROOT (struct rb_root) { NULL, }
 #define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL }
+#define RB_ROOT_LINKED (struct rb_root_linked) { {NULL, }, NULL }
 
 #endif

diff --git a/lib/rbtree.c b/lib/rbtree.c
index 18d42bcf4ec9d581807179f34561f4561900206d..5790d6ecba4ec9a75193079931daf8ec10c1f347 100644 (file)
--- a/lib/rbtree.c
+++ b/lib/rbtree.c
@@ -446,6 +446,23 @@ void rb_erase(struct rb_node *node, struct rb_root *root)
 }
 EXPORT_SYMBOL(rb_erase);
 
+bool rb_erase_linked(struct rb_node_linked *node, struct rb_root_linked *root)
+{
+       if (node->prev)
+               node->prev->next = node->next;
+       else
+               root->rb_leftmost = node->next;
+
+       if (node->next)
+               node->next->prev = node->prev;
+
+       rb_erase(&node->node, &root->rb_root);
+       RB_CLEAR_LINKED_NODE(node);
+
+       return !!root->rb_leftmost;
+}
+EXPORT_SYMBOL_GPL(rb_erase_linked);
+
 /*
  * Augmented rbtree manipulation functions.
  *