dma-buf: protected fence ops by RCU v8

The fence ops of a dma_fence currently need to life as long as the
dma_fence is alive. This means that the module which originally issued
a dma_fence can't unload unless all fences are freed up.

As first step to solve this issue protect the fence ops by RCU.

While it is counter intuitive to protect a constant function pointer table
by RCU it allows modules to wait for an RCU grace period before they
unload, to make sure that nobody is executing their functions any more.

This patch has not much functional change, but only adds the RCU
handling for the static checker to test.

v2: make one the now duplicated lockdep warnings a comment instead.
v3: Add more documentation to ->wait and ->release callback.
v4: fix typo in documentation
v5: rebased on drm-tip
v6: improve code comments
v7: improve commit message and code comments
v8: fix sparse rcu warnings

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@igalia.com>
Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
Link: https://lore.kernel.org/r/20260219160822.1529-2-christian.koenig@amd.com

diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 7e8db99186c2442188740205c0103599fd35906e..076e6e6c75bef40fab3f53ba702b312a983a8eb0 100644 (file)
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -522,6 +522,7 @@ EXPORT_SYMBOL(dma_fence_signal);
 signed long
 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 {
+       const struct dma_fence_ops *ops;
        signed long ret;
 
        if (WARN_ON(timeout < 0))
@@ -533,15 +534,22 @@ dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 
        dma_fence_enable_sw_signaling(fence);
 
-       if (trace_dma_fence_wait_start_enabled()) {
-               rcu_read_lock();
-               trace_dma_fence_wait_start(fence);
+       rcu_read_lock();
+       ops = rcu_dereference(fence->ops);
+       trace_dma_fence_wait_start(fence);
+       if (ops->wait) {
+               /*
+                * Implementing the wait ops is deprecated and not supported for
+                * issuers of fences who need their lifetime to be independent
+                * of their module after they signal, so it is ok to use the
+                * ops outside the RCU protected section.
+                */
+               rcu_read_unlock();
+               ret = ops->wait(fence, intr, timeout);
+       } else {
                rcu_read_unlock();
-       }
-       if (fence->ops->wait)
-               ret = fence->ops->wait(fence, intr, timeout);
-       else
                ret = dma_fence_default_wait(fence, intr, timeout);
+       }
        if (trace_dma_fence_wait_end_enabled()) {
                rcu_read_lock();
                trace_dma_fence_wait_end(fence);
@@ -562,6 +570,7 @@ void dma_fence_release(struct kref *kref)
 {
        struct dma_fence *fence =
                container_of(kref, struct dma_fence, refcount);
+       const struct dma_fence_ops *ops;
 
        rcu_read_lock();
        trace_dma_fence_destroy(fence);
@@ -593,12 +602,12 @@ void dma_fence_release(struct kref *kref)
                spin_unlock_irqrestore(fence->lock, flags);
        }
 
-       rcu_read_unlock();
-
-       if (fence->ops->release)
-               fence->ops->release(fence);
+       ops = rcu_dereference(fence->ops);
+       if (ops->release)
+               ops->release(fence);
        else
                dma_fence_free(fence);
+       rcu_read_unlock();
 }
 EXPORT_SYMBOL(dma_fence_release);
 
@@ -617,6 +626,7 @@ EXPORT_SYMBOL(dma_fence_free);
 
 static bool __dma_fence_enable_signaling(struct dma_fence *fence)
 {
+       const struct dma_fence_ops *ops;
        bool was_set;
 
        lockdep_assert_held(fence->lock);
@@ -627,14 +637,18 @@ static bool __dma_fence_enable_signaling(struct dma_fence *fence)
        if (dma_fence_test_signaled_flag(fence))
                return false;
 
-       if (!was_set && fence->ops->enable_signaling) {
+       rcu_read_lock();
+       ops = rcu_dereference(fence->ops);
+       if (!was_set && ops->enable_signaling) {
                trace_dma_fence_enable_signal(fence);
 
-               if (!fence->ops->enable_signaling(fence)) {
+               if (!ops->enable_signaling(fence)) {
+                       rcu_read_unlock();
                        dma_fence_signal_locked(fence);
                        return false;
                }
        }
+       rcu_read_unlock();
 
        return true;
 }
@@ -1007,8 +1021,13 @@ EXPORT_SYMBOL(dma_fence_wait_any_timeout);
  */
 void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
 {
-       if (fence->ops->set_deadline && !dma_fence_is_signaled(fence))
-               fence->ops->set_deadline(fence, deadline);
+       const struct dma_fence_ops *ops;
+
+       rcu_read_lock();
+       ops = rcu_dereference(fence->ops);
+       if (ops->set_deadline && !dma_fence_is_signaled(fence))
+               ops->set_deadline(fence, deadline);
+       rcu_read_unlock();
 }
 EXPORT_SYMBOL(dma_fence_set_deadline);
 
@@ -1049,7 +1068,13 @@ __dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
        BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
 
        kref_init(&fence->refcount);
-       fence->ops = ops;
+       /*
+        * While it is counter intuitive to protect a constant function pointer
+        * table by RCU it allows modules to wait for an RCU grace period
+        * before they unload, to make sure that nobody is executing their
+        * functions any more.
+        */
+       RCU_INIT_POINTER(fence->ops, ops);
        INIT_LIST_HEAD(&fence->cb_list);
        fence->lock = lock;
        fence->context = context;
@@ -1129,11 +1154,12 @@ EXPORT_SYMBOL(dma_fence_init64);
  */
 const char __rcu *dma_fence_driver_name(struct dma_fence *fence)
 {
-       RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
-                        "RCU protection is required for safe access to returned string");
+       const struct dma_fence_ops *ops;
 
+       /* RCU protection is required for safe access to returned string */
+       ops = rcu_dereference(fence->ops);
        if (!dma_fence_test_signaled_flag(fence))
-               return (const char __rcu *)fence->ops->get_driver_name(fence);
+               return (const char __rcu *)ops->get_driver_name(fence);
        else
                return (const char __rcu *)"detached-driver";
 }
@@ -1161,11 +1187,12 @@ EXPORT_SYMBOL(dma_fence_driver_name);
  */
 const char __rcu *dma_fence_timeline_name(struct dma_fence *fence)
 {
-       RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
-                        "RCU protection is required for safe access to returned string");
+       const struct dma_fence_ops *ops;
 
+       /* RCU protection is required for safe access to returned string */
+       ops = rcu_dereference(fence->ops);
        if (!dma_fence_test_signaled_flag(fence))
-               return (const char __rcu *)fence->ops->get_driver_name(fence);
+               return (const char __rcu *)ops->get_driver_name(fence);
        else
                return (const char __rcu *)"signaled-timeline";
 }

diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 90684f30a048322acf02cb7a5e5c37e43ce437e3..960fdc1cc6bab4ce35e7a8ce3f5aa23941283f6c 100644 (file)
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -158,7 +158,7 @@ static const struct dma_fence_ops drm_crtc_fence_ops;
 
 static struct drm_crtc *fence_to_crtc(struct dma_fence *fence)
 {
-       BUG_ON(fence->ops != &drm_crtc_fence_ops);
+       BUG_ON(rcu_access_pointer(fence->ops) != &drm_crtc_fence_ops);
        return container_of(fence->lock, struct drm_crtc, fence_lock);
 }
 

diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index 9391d6f0dc01d7a02cce6d5a96ac7482966a6c7d..a27786cb86fb671e58bc4148e1df071f8392faee 100644 (file)
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -195,10 +195,10 @@ static const struct dma_fence_ops drm_sched_fence_ops_finished = {
 
 struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f)
 {
-       if (f->ops == &drm_sched_fence_ops_scheduled)
+       if (rcu_access_pointer(f->ops) == &drm_sched_fence_ops_scheduled)
                return container_of(f, struct drm_sched_fence, scheduled);
 
-       if (f->ops == &drm_sched_fence_ops_finished)
+       if (rcu_access_pointer(f->ops) == &drm_sched_fence_ops_finished)
                return container_of(f, struct drm_sched_fence, finished);
 
        return NULL;

diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
index 9c4d2528923905c522aea0fccd8b7d2f0031be6d..fa3cfe3e98ac1e0b9d457a7c9de0aa8fd8a4504a 100644 (file)
--- a/include/linux/dma-fence.h
+++ b/include/linux/dma-fence.h
@@ -67,7 +67,7 @@ struct seq_file;
  */
 struct dma_fence {
        spinlock_t *lock;
-       const struct dma_fence_ops *ops;
+       const struct dma_fence_ops __rcu *ops;
        /*
         * We clear the callback list on kref_put so that by the time we
         * release the fence it is unused. No one should be adding to the
@@ -220,6 +220,10 @@ struct dma_fence_ops {
         * timed out. Can also return other error values on custom implementations,
         * which should be treated as if the fence is signaled. For example a hardware
         * lockup could be reported like that.
+        *
+        * Implementing this callback prevents the fence from detaching after
+        * signaling and so it is necessary for the module providing the
+        * dma_fence_ops to stay loaded as long as the dma_fence exists.
         */
        signed long (*wait)(struct dma_fence *fence,
                            bool intr, signed long timeout);
@@ -231,6 +235,13 @@ struct dma_fence_ops {
         * Can be called from irq context.  This callback is optional. If it is
         * NULL, then dma_fence_free() is instead called as the default
         * implementation.
+        *
+        * Implementing this callback prevents the fence from detaching after
+        * signaling and so it is necessary for the module providing the
+        * dma_fence_ops to stay loaded as long as the dma_fence exists.
+        *
+        * If the callback is implemented the memory backing the dma_fence
+        * object must be freed RCU safe.
         */
        void (*release)(struct dma_fence *fence);
 
@@ -454,13 +465,19 @@ dma_fence_test_signaled_flag(struct dma_fence *fence)
 static inline bool
 dma_fence_is_signaled_locked(struct dma_fence *fence)
 {
+       const struct dma_fence_ops *ops;
+
        if (dma_fence_test_signaled_flag(fence))
                return true;
 
-       if (fence->ops->signaled && fence->ops->signaled(fence)) {
+       rcu_read_lock();
+       ops = rcu_dereference(fence->ops);
+       if (ops->signaled && ops->signaled(fence)) {
+               rcu_read_unlock();
                dma_fence_signal_locked(fence);
                return true;
        }
+       rcu_read_unlock();
 
        return false;
 }
@@ -484,13 +501,19 @@ dma_fence_is_signaled_locked(struct dma_fence *fence)
 static inline bool
 dma_fence_is_signaled(struct dma_fence *fence)
 {
+       const struct dma_fence_ops *ops;
+
        if (dma_fence_test_signaled_flag(fence))
                return true;
 
-       if (fence->ops->signaled && fence->ops->signaled(fence)) {
+       rcu_read_lock();
+       ops = rcu_dereference(fence->ops);
+       if (ops->signaled && ops->signaled(fence)) {
+               rcu_read_unlock();
                dma_fence_signal(fence);
                return true;
        }
+       rcu_read_unlock();
 
        return false;
 }
@@ -695,7 +718,7 @@ extern const struct dma_fence_ops dma_fence_chain_ops;
  */
 static inline bool dma_fence_is_array(struct dma_fence *fence)
 {
-       return fence->ops == &dma_fence_array_ops;
+       return rcu_access_pointer(fence->ops) == &dma_fence_array_ops;
 }
 
 /**
@@ -706,7 +729,7 @@ static inline bool dma_fence_is_array(struct dma_fence *fence)
  */
 static inline bool dma_fence_is_chain(struct dma_fence *fence)
 {
-       return fence->ops == &dma_fence_chain_ops;
+       return rcu_access_pointer(fence->ops) == &dma_fence_chain_ops;
 }
 
 /**