Merge tag 'mm-stable-2023-02-20-13-37' of git://git.kernel.org/pub/scm/linux/kernel...

[thirdparty/linux.git] / include / linux / mm_types.h

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 56753d0f096d06591c5c2ff29b05b72fb212c7b3..0722859c36478d92f06455c6841144a7bfe5d207 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -598,7 +598,18 @@ struct mm_struct {
                 * &struct mm_struct is freed.
                 */
                atomic_t mm_count;
-
+#ifdef CONFIG_SCHED_MM_CID
+               /**
+                * @cid_lock: Protect cid bitmap updates vs lookups.
+                *
+                * Prevent situations where updates to the cid bitmap happen
+                * concurrently with lookups. Those can lead to situations
+                * where a lookup cannot find a free bit simply because it was
+                * unlucky enough to load, non-atomically, bitmap words as they
+                * were being concurrently updated by the updaters.
+                */
+               raw_spinlock_t cid_lock;
+#endif
 #ifdef CONFIG_MMU
                atomic_long_t pgtables_bytes;   /* size of all page tables */
 #endif
@@ -860,6 +871,36 @@ static inline void vma_iter_init(struct vma_iterator *vmi,
        mas_init(&vmi->mas, &mm->mm_mt, addr);
 }
 
+#ifdef CONFIG_SCHED_MM_CID
+/* Accessor for struct mm_struct's cidmask. */
+static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
+{
+       unsigned long cid_bitmap = (unsigned long)mm;
+
+       cid_bitmap += offsetof(struct mm_struct, cpu_bitmap);
+       /* Skip cpu_bitmap */
+       cid_bitmap += cpumask_size();
+       return (struct cpumask *)cid_bitmap;
+}
+
+static inline void mm_init_cid(struct mm_struct *mm)
+{
+       raw_spin_lock_init(&mm->cid_lock);
+       cpumask_clear(mm_cidmask(mm));
+}
+
+static inline unsigned int mm_cid_size(void)
+{
+       return cpumask_size();
+}
+#else /* CONFIG_SCHED_MM_CID */
+static inline void mm_init_cid(struct mm_struct *mm) { }
+static inline unsigned int mm_cid_size(void)
+{
+       return 0;
+}
+#endif /* CONFIG_SCHED_MM_CID */
+
 struct mmu_gather;
 extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
 extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);