sched/fair: Introduce and use the vruntime_cmp() and vruntime_op() wrappers for wrapp...

We have to be careful with vruntime comparisons and subtraction,
due to the possibility of wrapping, so we have macros like:

   #define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })

Which is used like this:

		if (vruntime_gt(min_vruntime, se, rse))
			se->min_vruntime = rse->min_vruntime;

Replace this with an easier to read pattern that uses the regular
arithmetics operators:

		if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime))
			se->min_vruntime = rse->min_vruntime;

Also replace vruntime subtractions with vruntime_op():

	-       delta = (s64)(sea->vruntime - seb->vruntime) +
	-               (s64)(cfs_rqb->zero_vruntime_fi - cfs_rqa->zero_vruntime_fi);
	+       delta = vruntime_op(sea->vruntime, "-", seb->vruntime) +
	+               vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi);

In the vruntime_cmp() and vruntime_op() macros use Use __builtin_strcmp(),
because of __HAVE_ARCH_STRCMP might turn off the compiler optimizations
we rely on here to catch usage bugs.

No change in functionality.

Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index dcbd995de46d18bab25a75a9a107a90384cd47d9..a8ac68d17c6d77f8ddc042661e8e0d48ab8247f5 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -524,10 +524,48 @@ void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
  * Scheduling class tree data structure manipulation methods:
  */
 
+extern void __BUILD_BUG_vruntime_cmp(void);
+
+/* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */
+
+#define vruntime_cmp(A, CMP_STR, B) ({                         \
+       int __res = 0;                                          \
+                                                               \
+       if (!__builtin_strcmp(CMP_STR, "<")) {                  \
+               __res = ((s64)((A)-(B)) < 0);                   \
+       } else if (!__builtin_strcmp(CMP_STR, "<=")) {          \
+               __res = ((s64)((A)-(B)) <= 0);                  \
+       } else if (!__builtin_strcmp(CMP_STR, ">")) {           \
+               __res = ((s64)((A)-(B)) > 0);                   \
+       } else if (!__builtin_strcmp(CMP_STR, ">=")) {          \
+               __res = ((s64)((A)-(B)) >= 0);                  \
+       } else {                                                \
+               /* Unknown operator throws linker error: */     \
+               __BUILD_BUG_vruntime_cmp();                     \
+       }                                                       \
+                                                               \
+       __res;                                                  \
+})
+
+extern void __BUILD_BUG_vruntime_op(void);
+
+#define vruntime_op(A, OP_STR, B) ({                           \
+       s64 __res = 0;                                          \
+                                                               \
+       if (!__builtin_strcmp(OP_STR, "-")) {                   \
+               __res = (s64)((A)-(B));                         \
+       } else {                                                \
+               /* Unknown operator throws linker error: */     \
+               __BUILD_BUG_vruntime_op();                      \
+       }                                                       \
+                                                               \
+       __res;                                          \
+})
+
+
 static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime)
 {
-       s64 delta = (s64)(vruntime - max_vruntime);
-       if (delta > 0)
+       if (vruntime_cmp(vruntime, ">", max_vruntime))
                max_vruntime = vruntime;
 
        return max_vruntime;
@@ -535,8 +573,7 @@ static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime)
 
 static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime)
 {
-       s64 delta = (s64)(vruntime - min_vruntime);
-       if (delta < 0)
+       if (vruntime_cmp(vruntime, "<", min_vruntime))
                min_vruntime = vruntime;
 
        return min_vruntime;
@@ -549,12 +586,12 @@ static inline bool entity_before(const struct sched_entity *a,
         * Tiebreak on vruntime seems unnecessary since it can
         * hardly happen.
         */
-       return (s64)(a->deadline - b->deadline) < 0;
+       return vruntime_cmp(a->deadline, "<", b->deadline);
 }
 
 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-       return (s64)(se->vruntime - cfs_rq->zero_vruntime);
+       return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime);
 }
 
 #define __node_2_se(node) \
@@ -732,7 +769,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
                load += weight;
        }
 
-       return avg >= (s64)(vruntime - cfs_rq->zero_vruntime) * load;
+       return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load;
 }
 
 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -743,7 +780,7 @@ int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
 static void update_zero_vruntime(struct cfs_rq *cfs_rq)
 {
        u64 vruntime = avg_vruntime(cfs_rq);
-       s64 delta = (s64)(vruntime - cfs_rq->zero_vruntime);
+       s64 delta = vruntime_op(vruntime, "-", cfs_rq->zero_vruntime);
 
        sum_w_vruntime_update(cfs_rq, delta);
 
@@ -770,13 +807,12 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
        return entity_before(__node_2_se(a), __node_2_se(b));
 }
 
-#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })
-
 static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node)
 {
        if (node) {
                struct sched_entity *rse = __node_2_se(node);
-               if (vruntime_gt(min_vruntime, se, rse))
+
+               if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime))
                        se->min_vruntime = rse->min_vruntime;
        }
 }
@@ -887,7 +923,7 @@ static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_enti
 
 static inline bool protect_slice(struct sched_entity *se)
 {
-       return ((s64)(se->vprot - se->vruntime) > 0);
+       return vruntime_cmp(se->vruntime, "<", se->vprot);
 }
 
 static inline void cancel_protect_slice(struct sched_entity *se)
@@ -1024,7 +1060,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se);
  */
 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-       if ((s64)(se->vruntime - se->deadline) < 0)
+       if (vruntime_cmp(se->vruntime, "<", se->deadline))
                return false;
 
        /*
@@ -13293,8 +13329,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b,
         * zero_vruntime_fi, which would have been updated in prior calls
         * to se_fi_update().
         */
-       delta = (s64)(sea->vruntime - seb->vruntime) +
-               (s64)(cfs_rqb->zero_vruntime_fi - cfs_rqa->zero_vruntime_fi);
+       delta = vruntime_op(sea->vruntime, "-", seb->vruntime) +
+               vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi);
 
        return delta > 0;
 }