RDMA/core: Add aging to FRMR pools

Add aging mechanism to handles of FRMR pools.
Keep the handles stored in FRMR pools for at least 1 minute for
application to reuse, destroy all handles which were not reused.

Add a new queue to each pool to accomplish that.
Upon aging trigger, destroy all FRMR handles from the new 'inactive'
queue and move all handles from the 'active' pool to the 'inactive' pool.
This ensures all destroyed handles were not reused for at least one aging
time period and were not held longer than 2 aging time periods.
Handles from the inactive queue will be popped only if the active queue is
empty.

Signed-off-by: Michael Guralnik <michaelgur@nvidia.com>
Reviewed-by: Yishai Hadas <yishaih@nvidia.com>
Signed-off-by: Edward Srouji <edwards@nvidia.com>
Link: https://patch.msgid.link/20260226-frmr_pools-v4-3-95360b54f15e@nvidia.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/core/frmr_pools.c b/drivers/infiniband/core/frmr_pools.c
index e08c8093a468727860b6ca30ab2ae2ad4cc2cf3d..1be7253d7afd2c7f897dfbec6077daafe4f4fab3 100644 (file)
--- a/drivers/infiniband/core/frmr_pools.c
+++ b/drivers/infiniband/core/frmr_pools.c
@@ -8,9 +8,12 @@
 #include <linux/sort.h>
 #include <linux/spinlock.h>
 #include <rdma/ib_verbs.h>
+#include <linux/timer.h>
 
 #include "frmr_pools.h"
 
+#define FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS 60
+
 static int push_handle_to_queue_locked(struct frmr_queue *queue, u32 handle)
 {
        u32 tmp = queue->ci % NUM_HANDLES_PER_PAGE;
@@ -80,17 +83,56 @@ static bool pop_frmr_handles_page(struct ib_frmr_pool *pool,
        return true;
 }
 
-static void destroy_frmr_pool(struct ib_device *device,
-                             struct ib_frmr_pool *pool)
+static void destroy_all_handles_in_queue(struct ib_device *device,
+                                        struct ib_frmr_pool *pool,
+                                        struct frmr_queue *queue)
 {
        struct ib_frmr_pools *pools = device->frmr_pools;
        struct frmr_handles_page *page;
        u32 count;
 
-       while (pop_frmr_handles_page(pool, &pool->queue, &page, &count)) {
+       while (pop_frmr_handles_page(pool, queue, &page, &count)) {
                pools->pool_ops->destroy_frmrs(device, page->handles, count);
                kfree(page);
        }
+}
+
+static void pool_aging_work(struct work_struct *work)
+{
+       struct ib_frmr_pool *pool = container_of(
+               to_delayed_work(work), struct ib_frmr_pool, aging_work);
+       struct ib_frmr_pools *pools = pool->device->frmr_pools;
+       bool has_work = false;
+
+       destroy_all_handles_in_queue(pool->device, pool, &pool->inactive_queue);
+
+       /* Move all pages from regular queue to inactive queue */
+       spin_lock(&pool->lock);
+       if (pool->queue.ci > 0) {
+               list_splice_tail_init(&pool->queue.pages_list,
+                                     &pool->inactive_queue.pages_list);
+               pool->inactive_queue.num_pages = pool->queue.num_pages;
+               pool->inactive_queue.ci = pool->queue.ci;
+
+               pool->queue.num_pages = 0;
+               pool->queue.ci = 0;
+               has_work = true;
+       }
+       spin_unlock(&pool->lock);
+
+       /* Reschedule if there are handles to age in next aging period */
+       if (has_work)
+               queue_delayed_work(
+                       pools->aging_wq, &pool->aging_work,
+                       secs_to_jiffies(FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS));
+}
+
+static void destroy_frmr_pool(struct ib_device *device,
+                             struct ib_frmr_pool *pool)
+{
+       cancel_delayed_work_sync(&pool->aging_work);
+       destroy_all_handles_in_queue(device, pool, &pool->queue);
+       destroy_all_handles_in_queue(device, pool, &pool->inactive_queue);
 
        kfree(pool);
 }
@@ -115,6 +157,11 @@ int ib_frmr_pools_init(struct ib_device *device,
        pools->rb_root = RB_ROOT;
        rwlock_init(&pools->rb_lock);
        pools->pool_ops = pool_ops;
+       pools->aging_wq = create_singlethread_workqueue("frmr_aging_wq");
+       if (!pools->aging_wq) {
+               kfree(pools);
+               return -ENOMEM;
+       }
 
        device->frmr_pools = pools;
        return 0;
@@ -142,6 +189,7 @@ void ib_frmr_pools_cleanup(struct ib_device *device)
        rbtree_postorder_for_each_entry_safe(pool, next, &pools->rb_root, node)
                destroy_frmr_pool(device, pool);
 
+       destroy_workqueue(pools->aging_wq);
        kfree(pools);
        device->frmr_pools = NULL;
 }
@@ -229,7 +277,10 @@ static struct ib_frmr_pool *create_frmr_pool(struct ib_device *device,
 
        memcpy(&pool->key, key, sizeof(*key));
        INIT_LIST_HEAD(&pool->queue.pages_list);
+       INIT_LIST_HEAD(&pool->inactive_queue.pages_list);
        spin_lock_init(&pool->lock);
+       INIT_DELAYED_WORK(&pool->aging_work, pool_aging_work);
+       pool->device = device;
 
        write_lock(&pools->rb_lock);
        existing = rb_find_add(&pool->node, &pools->rb_root, frmr_pool_cmp_add);
@@ -256,11 +307,17 @@ static int get_frmr_from_pool(struct ib_device *device,
 
        spin_lock(&pool->lock);
        if (pool->queue.ci == 0) {
-               spin_unlock(&pool->lock);
-               err = pools->pool_ops->create_frmrs(device, &pool->key, &handle,
-                                                   1);
-               if (err)
-                       return err;
+               if (pool->inactive_queue.ci > 0) {
+                       handle = pop_handle_from_queue_locked(
+                               &pool->inactive_queue);
+                       spin_unlock(&pool->lock);
+               } else {
+                       spin_unlock(&pool->lock);
+                       err = pools->pool_ops->create_frmrs(device, &pool->key,
+                                                           &handle, 1);
+                       if (err)
+                               return err;
+               }
        } else {
                handle = pop_handle_from_queue_locked(&pool->queue);
                spin_unlock(&pool->lock);
@@ -308,12 +365,21 @@ EXPORT_SYMBOL(ib_frmr_pool_pop);
 int ib_frmr_pool_push(struct ib_device *device, struct ib_mr *mr)
 {
        struct ib_frmr_pool *pool = mr->frmr.pool;
+       struct ib_frmr_pools *pools = device->frmr_pools;
+       bool schedule_aging = false;
        int ret;
 
        spin_lock(&pool->lock);
+       /* Schedule aging every time an empty pool becomes non-empty */
+       if (pool->queue.ci == 0)
+               schedule_aging = true;
        ret = push_handle_to_queue_locked(&pool->queue, mr->frmr.handle);
        spin_unlock(&pool->lock);
 
+       if (ret == 0 && schedule_aging)
+               queue_delayed_work(pools->aging_wq, &pool->aging_work,
+                       secs_to_jiffies(FRMR_POOLS_DEFAULT_AGING_PERIOD_SECS));
+
        return ret;
 }
 EXPORT_SYMBOL(ib_frmr_pool_push);

diff --git a/drivers/infiniband/core/frmr_pools.h b/drivers/infiniband/core/frmr_pools.h
index 0433db5061bd1c401a5feb2fa6114b73d92db9fe..a80789c876383e731d3bf13673a973775777a4b8 100644 (file)
--- a/drivers/infiniband/core/frmr_pools.h
+++ b/drivers/infiniband/core/frmr_pools.h
@@ -11,6 +11,7 @@
 #include <linux/spinlock_types.h>
 #include <linux/types.h>
 #include <asm/page.h>
+#include <linux/workqueue.h>
 
 #define NUM_HANDLES_PER_PAGE \
        ((PAGE_SIZE - sizeof(struct list_head)) / sizeof(u32))
@@ -37,12 +38,18 @@ struct ib_frmr_pool {
        /* Protect access to the queue */
        spinlock_t lock;
        struct frmr_queue queue;
+       struct frmr_queue inactive_queue;
+
+       struct delayed_work aging_work;
+       struct ib_device *device;
 };
 
 struct ib_frmr_pools {
        struct rb_root rb_root;
        rwlock_t rb_lock;
        const struct ib_frmr_pool_ops *pool_ops;
+
+       struct workqueue_struct *aging_wq;
 };
 
 #endif /* RDMA_CORE_FRMR_POOLS_H */