rseq: Cache CPU ID and MM CID values

In preparation for rewriting RSEQ exit to user space handling provide
storage to cache the CPU ID and MM CID values which were written to user
space. That prepares for a quick check, which avoids the update when
nothing changed.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251027084306.841964081@linutronix.de

diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index ab91b1e6bb4ab41360245e4ee23de511cf49dd14..d315a92afb363afc5caca60199520287fa00ce16 100644 (file)
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -57,6 +57,7 @@ static inline void rseq_virt_userspace_exit(void)
 static inline void rseq_reset(struct task_struct *t)
 {
        memset(&t->rseq, 0, sizeof(t->rseq));
+       t->rseq.ids.cpu_cid = ~0ULL;
 }
 
 static inline void rseq_execve(struct task_struct *t)
@@ -70,10 +71,12 @@ static inline void rseq_execve(struct task_struct *t)
  */
 static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
 {
-       if (clone_flags & CLONE_VM)
+       if (clone_flags & CLONE_VM) {
                rseq_reset(t);
-       else
+       } else {
                t->rseq = current->rseq;
+               t->rseq.ids.cpu_cid = ~0ULL;
+       }
 }
 
 #else /* CONFIG_RSEQ */

diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index f7a60c8eddc90412482b5d760b5cd04160091a49..40901b033b9280ff84b1c8f8865411be4073bb70 100644 (file)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -30,18 +30,39 @@ struct rseq_event {
        };
 };
 
+/**
+ * struct rseq_ids - Cache for ids, which need to be updated
+ * @cpu_cid:   Compound of @cpu_id and @mm_cid to make the
+ *             compiler emit a single compare on 64-bit
+ * @cpu_id:    The CPU ID which was written last to user space
+ * @mm_cid:    The MM CID which was written last to user space
+ *
+ * @cpu_id and @mm_cid are updated when the data is written to user space.
+ */
+struct rseq_ids {
+       union {
+               u64             cpu_cid;
+               struct {
+                       u32     cpu_id;
+                       u32     mm_cid;
+               };
+       };
+};
+
 /**
  * struct rseq_data - Storage for all rseq related data
  * @usrptr:    Pointer to the registered user space RSEQ memory
  * @len:       Length of the RSEQ region
  * @sig:       Signature of critial section abort IPs
  * @event:     Storage for event management
+ * @ids:       Storage for cached CPU ID and MM CID
  */
 struct rseq_data {
        struct rseq __user              *usrptr;
        u32                             len;
        u32                             sig;
        struct rseq_event               event;
+       struct rseq_ids                 ids;
 };
 
 #else /* CONFIG_RSEQ */

diff --git a/include/trace/events/rseq.h b/include/trace/events/rseq.h
index 823b47d1ba1e14801ac687566d34c23b088786ba..ce85d650bf4b938590db2a1a4486914fa2638859 100644 (file)
--- a/include/trace/events/rseq.h
+++ b/include/trace/events/rseq.h
@@ -21,9 +21,9 @@ TRACE_EVENT(rseq_update,
        ),
 
        TP_fast_assign(
-               __entry->cpu_id = raw_smp_processor_id();
+               __entry->cpu_id = t->rseq.ids.cpu_id;
                __entry->node_id = cpu_to_node(__entry->cpu_id);
-               __entry->mm_cid = task_mm_cid(t);
+               __entry->mm_cid = t->rseq.ids.mm_cid;
        ),
 
        TP_printk("cpu_id=%d node_id=%d mm_cid=%d", __entry->cpu_id,

diff --git a/kernel/rseq.c b/kernel/rseq.c
index aae62661e6bb3f52b22d70366c3d9d9d6f939a11..ad1e7cecd527df6bfa5f99f49c61969ab7b5cbb9 100644 (file)
--- a/kernel/rseq.c
+++ b/kernel/rseq.c
@@ -184,6 +184,10 @@ static int rseq_update_cpu_node_id(struct task_struct *t)
        rseq_unsafe_put_user(t, node_id, node_id, efault_end);
        rseq_unsafe_put_user(t, mm_cid, mm_cid, efault_end);
 
+       /* Cache the user space values */
+       t->rseq.ids.cpu_id = cpu_id;
+       t->rseq.ids.mm_cid = mm_cid;
+
        /*
         * Additional feature fields added after ORIG_RSEQ_SIZE
         * need to be conditionally updated only if