Merge branch 'sched/cache'

Merge the cache aware balancer topic branch.

# Conflicts:
#	kernel/sched/topology.c

diff --cc Documentation/admin-guide/kernel-parameters.txt
Simple merge

diff --cc arch/x86/include/asm/processor.h
Simple merge

diff --cc include/linux/mm_types.h
Simple merge

diff --cc include/linux/sched.h
Simple merge

diff --cc init/Kconfig
Simple merge

diff --cc kernel/exit.c
Simple merge

diff --cc kernel/fork.c
Simple merge

diff --cc kernel/sched/core.c
Simple merge

diff --cc kernel/sched/fair.c
Simple merge

diff --cc kernel/sched/sched.h
index bfb4b47c021b2a929dd23503408f148ea8df3868,45a3b77f46aa089f00f0b5590387c9949eaa93aa..8eb8f83db6b04c0db1128bcdbf85ab26a3843bec
--- 1/kernel/sched/sched.h
--- 2/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@@ -2164,7 -2186,7 +2191,8 @@@ DECLARE_PER_CPU(struct sched_domain __r
  DECLARE_PER_CPU(int, sd_llc_size);
  DECLARE_PER_CPU(int, sd_llc_id);
  DECLARE_PER_CPU(int, sd_share_id);
+ DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
 +DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
  DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
  DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
  DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);

diff --cc kernel/sched/topology.c
index f96d50131495abf99b80fa5817e2bbf380505da4,e47a3f72eb72be057ea70a8e91de3a2febaadb44..dbfd9657f8974a4705388efd4365134b43f2f43b
--- 1/kernel/sched/topology.c
--- 2/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@@ -663,9 -670,9 +670,10 @@@ static void destroy_sched_domains(struc
   */
  DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
  DEFINE_PER_CPU(int, sd_llc_size);
- DEFINE_PER_CPU(int, sd_llc_id);
+ DEFINE_PER_CPU(int, sd_llc_id) = -1;
  DEFINE_PER_CPU(int, sd_share_id);
+ DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
 +DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
  DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
  DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
  DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
@@@ -729,6 -717,9 +718,20 @@@ static void update_top_cache_domain(in
  
        sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
        rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd);
+ 
+       sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
++      /*
++       * The shared object is attached to sd_asym_cpucapacity only when the
++       * asym domain is non-overlapping (i.e., not built from SD_NUMA).
++       * On overlapping (NUMA) asym domains we fall back to letting the
++       * SD_SHARE_LLC path own the shared object, so sd->shared may be NULL
++       * here.
++       */
++      if (sd && sd->shared)
++              sds = sd->shared;
++
+       rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
++      rcu_assign_pointer(per_cpu(sd_balance_shared, cpu), sds);
  }
  
  /*
@@@ -2663,54 -2906,61 +2916,109 @@@ static void adjust_numa_imbalance(struc
        }
  }
  
 +static void init_sched_domain_shared(struct s_data *d, struct sched_domain *sd)
 +{
 +      int sd_id = cpumask_first(sched_domain_span(sd));
 +
 +      sd->shared = *per_cpu_ptr(d->sds, sd_id);
 +      /*
 +       * nr_busy_cpus is consumed only by the NOHZ kick path via
 +       * sd_balance_shared; on the asym-capacity path it is initialized but
 +       * never read.
 +       */
 +      atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
 +      atomic_inc(&sd->shared->ref);
 +}
 +
 +/*
 + * For asymmetric CPU capacity, attach sched_domain_shared on the innermost
 + * SD_ASYM_CPUCAPACITY_FULL ancestor of @cpu's base domain when that ancestor is
 + * not an overlapping NUMA-built domain (then LLC should claim shared).
 + *
 + * A CPU may lack any FULL ancestor (e.g., exclusive cpuset symmetric island),
 + * then LLC must claim shared instead.
 + *
 + * Note: SD_ASYM_CPUCAPACITY_FULL is only set when all CPU capacity values
 + * are present in the domain span, so the asym domain we attach to cannot
 + * degenerate into a single-capacity group. The relevant edge cases are instead
 + * covered by the caveats above.
 + *
 + * Return true if this CPU's asym path claimed sd->shared, false otherwise.
 + */
 +static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu)
 +{
 +      struct sched_domain *sd = *per_cpu_ptr(d->sd, cpu);
 +      struct sched_domain *sd_asym;
 +
 +      if (!sd)
 +              return false;
 +
 +      sd_asym = sd;
 +      while (sd_asym && !(sd_asym->flags & SD_ASYM_CPUCAPACITY_FULL))
 +              sd_asym = sd_asym->parent;
 +
 +      if (!sd_asym || (sd_asym->flags & SD_NUMA))
 +              return false;
 +
 +      init_sched_domain_shared(d, sd_asym);
 +      return true;
 +}
 +
+ static int __sched_domains_alloc_llc_id(void)
+ {
+       int lid, max;
+ 
+       lockdep_assert_held(&sched_domains_mutex);
+ 
+       lid = cpumask_first_zero(sched_domains_llc_id_allocmask);
+       /*
+        * llc_id space should never grow larger than the
+        * possible number of CPUs in the system.
+        */
+       if (lid >= nr_cpu_ids)
+               return -1;
+ 
+       __cpumask_set_cpu(lid, sched_domains_llc_id_allocmask);
+       max = cpumask_last(sched_domains_llc_id_allocmask);
+       if (max > max_lid)
+               max_lid = max;
+ 
+       return lid;
+ }
+ 
+ static void __sched_domains_free_llc_id(int cpu)
+ {
+       int i, lid, max;
+ 
+       lockdep_assert_held(&sched_domains_mutex);
+ 
+       lid = per_cpu(sd_llc_id, cpu);
+       if (lid == -1 || lid >= nr_cpu_ids)
+               return;
+ 
+       per_cpu(sd_llc_id, cpu) = -1;
+ 
+       for_each_cpu(i, llc_mask(cpu)) {
+               /* An online CPU owns the llc_id. */
+               if (per_cpu(sd_llc_id, i) == lid)
+                       return;
+       }
+ 
+       __cpumask_clear_cpu(lid, sched_domains_llc_id_allocmask);
+ 
+       max = cpumask_last(sched_domains_llc_id_allocmask);
+       /* shrink max lid to save memory */
+       if (max < max_lid)
+               max_lid = max;
+ }
+ 
+ void sched_domains_free_llc_id(int cpu)
+ {
+       sched_domains_mutex_lock();
+       __sched_domains_free_llc_id(cpu);
+       sched_domains_mutex_unlock();
+ }
+ 
  /*
   * Build sched domains for a given set of CPUs and attach the sched domains
   * to the individual CPUs
@@@ -2775,20 -3049,16 +3107,15 @@@ build_sched_domains(const struct cpumas
                if (!sd)
                        continue;
  
-                       asym_claimed = claim_asym_sched_domain_shared(&d, i);
 +              if (has_asym)
++                      claim_asym_sched_domain_shared(&d, i);
 +
                /* First, find the topmost SD_SHARE_LLC domain */
                while (sd->parent && (sd->parent->flags & SD_SHARE_LLC))
                        sd = sd->parent;
  
                if (sd->flags & SD_SHARE_LLC) {
-                       /*
-                        * Initialize the sd->shared for SD_SHARE_LLC unless
-                        * the asym path above already claimed it.
-                        */
-                       if (!asym_claimed)
-                               init_sched_domain_shared(&d, sd);
 -                      int sd_id = cpumask_first(sched_domain_span(sd));
 -
 -                      sd->shared = *per_cpu_ptr(d.sds, sd_id);
 -                      atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
 -                      atomic_inc(&sd->shared->ref);
++                      init_sched_domain_shared(&d, sd);
  
                        /*
                         * In presence of higher domains, adjust the