kernel/sched/isolation.c

   1 /*
   2  *  Housekeeping management. Manage the targets for routine code that can run on
   3  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   4  *
   5  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   6  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   7  *
   8  */
   9 #include "sched.h"
  10
  11 DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
  12 EXPORT_SYMBOL_GPL(housekeeping_overriden);
  13 static cpumask_var_t housekeeping_mask;
  14 static unsigned int housekeeping_flags;
  15
  16 int housekeeping_any_cpu(enum hk_flags flags)
  17 {
  18         if (static_branch_unlikely(&housekeeping_overriden))
  19                 if (housekeeping_flags & flags)
  20                         return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  21         return smp_processor_id();
  22 }
  23 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  24
  25 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  26 {
  27         if (static_branch_unlikely(&housekeeping_overriden))
  28                 if (housekeeping_flags & flags)
  29                         return housekeeping_mask;
  30         return cpu_possible_mask;
  31 }
  32 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  33
  34 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  35 {
  36         if (static_branch_unlikely(&housekeeping_overriden))
  37                 if (housekeeping_flags & flags)
  38                         set_cpus_allowed_ptr(t, housekeeping_mask);
  39 }
  40 EXPORT_SYMBOL_GPL(housekeeping_affine);
  41
  42 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  43 {
  44         if (static_branch_unlikely(&housekeeping_overriden))
  45                 if (housekeeping_flags & flags)
  46                         return cpumask_test_cpu(cpu, housekeeping_mask);
  47         return true;
  48 }
  49 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  50
  51 void __init housekeeping_init(void)
  52 {
  53         if (!housekeeping_flags)
  54                 return;
  55
  56         static_branch_enable(&housekeeping_overriden);
  57
  58         if (housekeeping_flags & HK_FLAG_TICK)
  59                 sched_tick_offload_init();
  60
  61         /* We need at least one CPU to handle housekeeping work */
  62         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  63 }
  64
  65 static int __init housekeeping_setup(char *str, enum hk_flags flags)
  66 {
  67         cpumask_var_t non_housekeeping_mask;
  68         int err;
  69
  70         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  71         err = cpulist_parse(str, non_housekeeping_mask);
  72         if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  73                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  74                 free_bootmem_cpumask_var(non_housekeeping_mask);
  75                 return 0;
  76         }
  77
  78         if (!housekeeping_flags) {
  79                 alloc_bootmem_cpumask_var(&housekeeping_mask);
  80                 cpumask_andnot(housekeeping_mask,
  81                                cpu_possible_mask, non_housekeeping_mask);
  82                 if (cpumask_empty(housekeeping_mask))
  83                         cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
  84         } else {
  85                 cpumask_var_t tmp;
  86
  87                 alloc_bootmem_cpumask_var(&tmp);
  88                 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
  89                 if (!cpumask_equal(tmp, housekeeping_mask)) {
  90                         pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
  91                         free_bootmem_cpumask_var(tmp);
  92                         free_bootmem_cpumask_var(non_housekeeping_mask);
  93                         return 0;
  94                 }
  95                 free_bootmem_cpumask_var(tmp);
  96         }
  97
  98         if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
  99                 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 100                         tick_nohz_full_setup(non_housekeeping_mask);
 101                 } else {
 102                         pr_warn("Housekeeping: nohz unsupported."
 103                                 " Build with CONFIG_NO_HZ_FULL\n");
 104                         free_bootmem_cpumask_var(non_housekeeping_mask);
 105                         return 0;
 106                 }
 107         }
 108
 109         housekeeping_flags |= flags;
 110
 111         free_bootmem_cpumask_var(non_housekeeping_mask);
 112
 113         return 1;
 114 }
 115
 116 static int __init housekeeping_nohz_full_setup(char *str)
 117 {
 118         unsigned int flags;
 119
 120         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
 121
 122         return housekeeping_setup(str, flags);
 123 }
 124 __setup("nohz_full=", housekeeping_nohz_full_setup);
 125
 126 static int __init housekeeping_isolcpus_setup(char *str)
 127 {
 128         unsigned int flags = 0;
 129
 130         while (isalpha(*str)) {
 131                 if (!strncmp(str, "nohz,", 5)) {
 132                         str += 5;
 133                         flags |= HK_FLAG_TICK;
 134                         continue;
 135                 }
 136
 137                 if (!strncmp(str, "domain,", 7)) {
 138                         str += 7;
 139                         flags |= HK_FLAG_DOMAIN;
 140                         continue;
 141                 }
 142
 143                 pr_warn("isolcpus: Error, unknown flag\n");
 144                 return 0;
 145         }
 146
 147         /* Default behaviour for isolcpus without flags */
 148         if (!flags)
 149                 flags |= HK_FLAG_DOMAIN;
 150
 151         return housekeeping_setup(str, flags);
 152 }
 153 __setup("isolcpus=", housekeeping_isolcpus_setup);