cpumask: Introduce cpumask_weighted_or()

CID management OR's two cpumasks and then calculates the weight on the
result. That's inefficient as that has to walk the same stuff twice. As
this is done with runqueue lock held, there is a real benefit of speeding
this up. Depending on the system this results in 10-20% less cycles spent
with runqueue lock held for a 4K cpumask.

Provide cpumask_weighted_or() and the corresponding bitmap functions which
return the weight of the OR result right away.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.448263340@linutronix.de

diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index 595217b7a6e71d84ed41114e22c0f15be4f3decb..b0395e4ccf9036d593a7b818297f9658f3c402ab 100644 (file)
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -45,6 +45,7 @@ struct device;
  *  bitmap_copy(dst, src, nbits)                *dst = *src
  *  bitmap_and(dst, src1, src2, nbits)          *dst = *src1 & *src2
  *  bitmap_or(dst, src1, src2, nbits)           *dst = *src1 | *src2
+ *  bitmap_weighted_or(dst, src1, src2, nbits) *dst = *src1 | *src2. Returns Hamming Weight of dst
  *  bitmap_xor(dst, src1, src2, nbits)          *dst = *src1 ^ *src2
  *  bitmap_andnot(dst, src1, src2, nbits)       *dst = *src1 & ~(*src2)
  *  bitmap_complement(dst, src, nbits)          *dst = ~(*src)
@@ -165,6 +166,8 @@ bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
                 const unsigned long *bitmap2, unsigned int nbits);
 void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
                 const unsigned long *bitmap2, unsigned int nbits);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+                                 const unsigned long *bitmap2, unsigned int nbits);
 void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
                  const unsigned long *bitmap2, unsigned int nbits);
 bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
@@ -337,6 +340,18 @@ void bitmap_or(unsigned long *dst, const unsigned long *src1,
                __bitmap_or(dst, src1, src2, nbits);
 }
 
+static __always_inline
+unsigned int bitmap_weighted_or(unsigned long *dst, const unsigned long *src1,
+                               const unsigned long *src2, unsigned int nbits)
+{
+       if (small_const_nbits(nbits)) {
+               *dst = *src1 | *src2;
+               return hweight_long(*dst & BITMAP_LAST_WORD_MASK(nbits));
+       } else {
+               return __bitmap_weighted_or(dst, src1, src2, nbits);
+       }
+}
+
 static __always_inline
 void bitmap_xor(unsigned long *dst, const unsigned long *src1,
                const unsigned long *src2, unsigned int nbits)

diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index ff8f41ab7ce63597fcf80ce4d34a9f228702c349..feba06eb0a42e77ae8cadac1b28356ddec77d392 100644 (file)
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -728,6 +728,22 @@ void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
                                      cpumask_bits(src2p), small_cpumask_bits);
 }
 
+/**
+ * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result
+ * @dstp: the cpumask result
+ * @src1p: the first input
+ * @src2p: the second input
+ *
+ * Return: The number of bits set in the resulting cpumask @dstp
+ */
+static __always_inline
+unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p,
+                                const struct cpumask *src2p)
+{
+       return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p),
+                                 cpumask_bits(src2p), small_cpumask_bits);
+}
+
 /**
  * cpumask_xor - *dstp = *src1p ^ *src2p
  * @dstp: the cpumask result

diff --git a/lib/bitmap.c b/lib/bitmap.c
index b97692854966e0073e32dfd4589b42ae3359c9cb..9dc526507875b7cbd427e860bb589a1c9f33ad1b 100644 (file)
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -355,6 +355,12 @@ unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1,
 }
 EXPORT_SYMBOL(__bitmap_weight_andnot);
 
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+                                 const unsigned long *bitmap2, unsigned int bits)
+{
+       return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits);
+}
+
 void __bitmap_set(unsigned long *map, unsigned int start, int len)
 {
        unsigned long *p = map + BIT_WORD(start);