rseq: Implement read only ABI enforcement for optimized RSEQ V2 mode

The optimized RSEQ V2 mode requires that user space adheres to the ABI
specification and does not modify the read-only fields cpu_id_start,
cpu_id, node_id and mm_cid behind the kernel's back.

While the kernel does not rely on these fields, the adherence to this is a
fundamental prerequisite to allow multiple entities, e.g. libraries, in an
application to utilize the full potential of RSEQ without stepping on each
other toes.

Validate this adherence on every update of these fields. If the kernel
detects that user space modified the fields, the application is force
terminated.

Fixes: d6200245c75e ("rseq: Allow registering RSEQ with slice extension")
Signed-off-by: Thomas Gleixner <tglx@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Dmitry Vyukov <dvyukov@google.com>
Tested-by: Dmitry Vyukov <dvyukov@google.com>
Link: https://patch.msgid.link/20260428224427.845230956%40kernel.org
Cc: stable@vger.kernel.org

diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h
index 934db41ec782611de7fb648e25ee6b9078640d63..2d0295df5107c768e0e811b61f692a314945fc23 100644 (file)
--- a/include/linux/rseq_entry.h
+++ b/include/linux/rseq_entry.h
@@ -248,7 +248,6 @@ static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, un
 #endif /* !CONFIG_RSEQ_SLICE_EXTENSION */
 
 bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr);
-bool rseq_debug_validate_ids(struct task_struct *t);
 
 static __always_inline void rseq_note_user_irq_entry(void)
 {
@@ -368,43 +367,6 @@ efault:
        return false;
 }
 
-/*
- * On debug kernels validate that user space did not mess with it if the
- * debug branch is enabled.
- */
-bool rseq_debug_validate_ids(struct task_struct *t)
-{
-       struct rseq __user *rseq = t->rseq.usrptr;
-       u32 cpu_id, uval, node_id;
-
-       /*
-        * On the first exit after registering the rseq region CPU ID is
-        * RSEQ_CPU_ID_UNINITIALIZED and node_id in user space is 0!
-        */
-       node_id = t->rseq.ids.cpu_id != RSEQ_CPU_ID_UNINITIALIZED ?
-                 cpu_to_node(t->rseq.ids.cpu_id) : 0;
-
-       scoped_user_read_access(rseq, efault) {
-               unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault);
-               if (cpu_id != t->rseq.ids.cpu_id)
-                       goto die;
-               unsafe_get_user(uval, &rseq->cpu_id, efault);
-               if (uval != cpu_id)
-                       goto die;
-               unsafe_get_user(uval, &rseq->node_id, efault);
-               if (uval != node_id)
-                       goto die;
-               unsafe_get_user(uval, &rseq->mm_cid, efault);
-               if (uval != t->rseq.ids.mm_cid)
-                       goto die;
-       }
-       return true;
-die:
-       t->rseq.event.fatal = true;
-efault:
-       return false;
-}
-
 #endif /* RSEQ_BUILD_SLOW_PATH */
 
 /*
@@ -514,20 +476,32 @@ efault:
  * faults in task context are fatal too.
  */
 static rseq_inline
-bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids,
-                            u32 node_id, u64 *csaddr)
+bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids, u64 *csaddr)
 {
        struct rseq __user *rseq = t->rseq.usrptr;
 
-       if (static_branch_unlikely(&rseq_debug_enabled)) {
-               if (!rseq_debug_validate_ids(t))
-                       return false;
-       }
-
        scoped_user_rw_access(rseq, efault) {
+               /* Validate the R/O fields for debug and optimized mode */
+               if (static_branch_unlikely(&rseq_debug_enabled) || rseq_v2(t)) {
+                       u32 cpu_id, uval;
+
+                       unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault);
+                       if (cpu_id != t->rseq.ids.cpu_id)
+                               goto die;
+                       unsafe_get_user(uval, &rseq->cpu_id, efault);
+                       if (uval != cpu_id)
+                               goto die;
+                       unsafe_get_user(uval, &rseq->node_id, efault);
+                       if (uval != t->rseq.ids.node_id)
+                               goto die;
+                       unsafe_get_user(uval, &rseq->mm_cid, efault);
+                       if (uval != t->rseq.ids.mm_cid)
+                               goto die;
+               }
+
                unsafe_put_user(ids->cpu_id, &rseq->cpu_id_start, efault);
                unsafe_put_user(ids->cpu_id, &rseq->cpu_id, efault);
-               unsafe_put_user(node_id, &rseq->node_id, efault);
+               unsafe_put_user(ids->node_id, &rseq->node_id, efault);
                unsafe_put_user(ids->mm_cid, &rseq->mm_cid, efault);
                if (csaddr)
                        unsafe_get_user(*csaddr, &rseq->rseq_cs, efault);
@@ -539,10 +513,13 @@ bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids,
 
        rseq_slice_clear_grant(t);
        /* Cache the new values */
-       t->rseq.ids.cpu_cid = ids->cpu_cid;
+       t->rseq.ids = *ids;
        rseq_stat_inc(rseq_stats.ids);
        rseq_trace_update(t, ids);
        return true;
+
+die:
+       t->rseq.event.fatal = true;
 efault:
        return false;
 }
@@ -552,11 +529,11 @@ efault:
  * is in a critical section.
  */
 static rseq_inline bool rseq_update_usr(struct task_struct *t, struct pt_regs *regs,
-                                       struct rseq_ids *ids, u32 node_id)
+                                       struct rseq_ids *ids)
 {
        u64 csaddr;
 
-       if (!rseq_set_ids_get_csaddr(t, ids, node_id, &csaddr))
+       if (!rseq_set_ids_get_csaddr(t, ids, &csaddr))
                return false;
 
        /*
@@ -659,12 +636,12 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t
        }
 
        struct rseq_ids ids = {
-               .cpu_id = task_cpu(t),
-               .mm_cid = task_mm_cid(t),
+               .cpu_id  = task_cpu(t),
+               .mm_cid  = task_mm_cid(t),
+               .node_id = cpu_to_node(ids.cpu_id),
        };
-       u32 node_id = cpu_to_node(ids.cpu_id);
 
-       return rseq_update_usr(t, regs, &ids, node_id);
+       return rseq_update_usr(t, regs, &ids);
 efault:
        return false;
 }

diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index a469c1870849c44365bf017efbae74f251f7b0e0..85739a63e85e6f2784716da2d91c3a2588367865 100644 (file)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -66,8 +66,9 @@ struct rseq_event {
  *             compiler emit a single compare on 64-bit
  * @cpu_id:    The CPU ID which was written last to user space
  * @mm_cid:    The MM CID which was written last to user space
+ * @node_id:   The node ID which was written last to user space
  *
- * @cpu_id and @mm_cid are updated when the data is written to user space.
+ * @cpu_id, @mm_cid and @node_id are updated when the data is written to user space.
  */
 struct rseq_ids {
        union {
@@ -77,6 +78,7 @@ struct rseq_ids {
                        u32     mm_cid;
                };
        };
+       u32                     node_id;
 };
 
 /**

diff --git a/kernel/rseq.c b/kernel/rseq.c
index aa25753ea1350cbb274025e43a00f2c1c89f31a3..101612027f6a3657d1ee9db4d7fa07071939c39f 100644 (file)
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -263,7 +263,6 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs)
        };
        struct task_struct *t = current;
        struct rseq_ids ids;
-       u32 node_id;
        bool event;
 
        if (unlikely(t->flags & PF_EXITING))
@@ -299,9 +298,9 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs)
        if (!event)
                return;
 
-       node_id = cpu_to_node(ids.cpu_id);
+       ids.node_id = cpu_to_node(ids.cpu_id);
 
-       if (unlikely(!rseq_update_usr(t, regs, &ids, node_id))) {
+       if (unlikely(!rseq_update_usr(t, regs, &ids))) {
                /*
                 * Clear the errors just in case this might survive magically, but
                 * leave the rest intact.