2 * Resource Director Technology(RDT)
5 * Copyright (C) 2017 Intel Corporation
8 * Vikas Shivappa <vikas.shivappa@intel.com>
10 * This replaces the cqm.c based on perf but we reuse a lot of
11 * code and datastructures originally from Peter Zijlstra and Matt Fleming.
13 * This program is free software; you can redistribute it and/or modify it
14 * under the terms and conditions of the GNU General Public License,
15 * version 2, as published by the Free Software Foundation.
17 * This program is distributed in the hope it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
22 * More information about RDT be found in the Intel (R) x86 Architecture
23 * Software Developer Manual June 2016, volume 3, section 17.17.
26 #include <linux/module.h>
27 #include <linux/slab.h>
28 #include <asm/cpu_device_id.h>
34 struct list_head list
;
38 * @rmid_free_lru A least recently used list of free RMIDs
39 * These RMIDs are guaranteed to have an occupancy less than the
42 static LIST_HEAD(rmid_free_lru
);
45 * @rmid_limbo_count count of currently unused but (potentially)
47 * This counts RMIDs that no one is currently using but that
48 * may have a occupancy value > intel_cqm_threshold. User can change
49 * the threshold occupancy value.
51 static unsigned int rmid_limbo_count
;
54 * @rmid_entry - The entry in the limbo and free lists.
56 static struct rmid_entry
*rmid_ptrs
;
59 * Global boolean for rdt_monitor which is true if any
60 * resource monitoring is enabled.
65 * Global to indicate which monitoring events are enabled.
67 unsigned int rdt_mon_features
;
70 * This is the threshold cache occupancy at which we will consider an
71 * RMID available for re-allocation.
73 unsigned int resctrl_cqm_threshold
;
75 static inline struct rmid_entry
*__rmid_entry(u32 rmid
)
77 struct rmid_entry
*entry
;
79 entry
= &rmid_ptrs
[rmid
];
80 WARN_ON(entry
->rmid
!= rmid
);
85 static u64
__rmid_read(u32 rmid
, u32 eventid
)
90 * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured
91 * with a valid event code for supported resource type and the bits
92 * IA32_QM_EVTSEL.RMID (bits 41:32) are configured with valid RMID,
93 * IA32_QM_CTR.data (bits 61:0) reports the monitored data.
94 * IA32_QM_CTR.Error (bit 63) and IA32_QM_CTR.Unavailable (bit 62)
97 wrmsr(MSR_IA32_QM_EVTSEL
, eventid
, rmid
);
98 rdmsrl(MSR_IA32_QM_CTR
, val
);
103 static bool rmid_dirty(struct rmid_entry
*entry
)
105 u64 val
= __rmid_read(entry
->rmid
, QOS_L3_OCCUP_EVENT_ID
);
107 return val
>= resctrl_cqm_threshold
;
111 * Check the RMIDs that are marked as busy for this domain. If the
112 * reported LLC occupancy is below the threshold clear the busy bit and
113 * decrement the count. If the busy count gets to zero on an RMID, we
116 void __check_limbo(struct rdt_domain
*d
, bool force_free
)
118 struct rmid_entry
*entry
;
119 struct rdt_resource
*r
;
120 u32 crmid
= 1, nrmid
;
122 r
= &rdt_resources_all
[RDT_RESOURCE_L3
];
125 * Skip RMID 0 and start from RMID 1 and check all the RMIDs that
126 * are marked as busy for occupancy < threshold. If the occupancy
127 * is less than the threshold decrement the busy counter of the
128 * RMID and move it to the free list when the counter reaches 0.
131 nrmid
= find_next_bit(d
->rmid_busy_llc
, r
->num_rmid
, crmid
);
132 if (nrmid
>= r
->num_rmid
)
135 entry
= __rmid_entry(nrmid
);
136 if (force_free
|| !rmid_dirty(entry
)) {
137 clear_bit(entry
->rmid
, d
->rmid_busy_llc
);
138 if (!--entry
->busy
) {
140 list_add_tail(&entry
->list
, &rmid_free_lru
);
147 bool has_busy_rmid(struct rdt_resource
*r
, struct rdt_domain
*d
)
149 return find_first_bit(d
->rmid_busy_llc
, r
->num_rmid
) != r
->num_rmid
;
153 * As of now the RMIDs allocation is global.
154 * However we keep track of which packages the RMIDs
155 * are used to optimize the limbo list management.
159 struct rmid_entry
*entry
;
161 lockdep_assert_held(&rdtgroup_mutex
);
163 if (list_empty(&rmid_free_lru
))
164 return rmid_limbo_count
? -EBUSY
: -ENOSPC
;
166 entry
= list_first_entry(&rmid_free_lru
,
167 struct rmid_entry
, list
);
168 list_del(&entry
->list
);
173 static void add_rmid_to_limbo(struct rmid_entry
*entry
)
175 struct rdt_resource
*r
;
176 struct rdt_domain
*d
;
180 r
= &rdt_resources_all
[RDT_RESOURCE_L3
];
184 list_for_each_entry(d
, &r
->domains
, list
) {
185 if (cpumask_test_cpu(cpu
, &d
->cpu_mask
)) {
186 val
= __rmid_read(entry
->rmid
, QOS_L3_OCCUP_EVENT_ID
);
187 if (val
<= resctrl_cqm_threshold
)
192 * For the first limbo RMID in the domain,
193 * setup up the limbo worker.
195 if (!has_busy_rmid(r
, d
))
196 cqm_setup_limbo_handler(d
, CQM_LIMBOCHECK_INTERVAL
);
197 set_bit(entry
->rmid
, d
->rmid_busy_llc
);
205 list_add_tail(&entry
->list
, &rmid_free_lru
);
208 void free_rmid(u32 rmid
)
210 struct rmid_entry
*entry
;
215 lockdep_assert_held(&rdtgroup_mutex
);
217 entry
= __rmid_entry(rmid
);
219 if (is_llc_occupancy_enabled())
220 add_rmid_to_limbo(entry
);
222 list_add_tail(&entry
->list
, &rmid_free_lru
);
225 static u64
mbm_overflow_count(u64 prev_msr
, u64 cur_msr
)
227 u64 shift
= 64 - MBM_CNTR_WIDTH
, chunks
;
229 chunks
= (cur_msr
<< shift
) - (prev_msr
<< shift
);
230 return chunks
>>= shift
;
233 static int __mon_event_count(u32 rmid
, struct rmid_read
*rr
)
238 tval
= __rmid_read(rmid
, rr
->evtid
);
239 if (tval
& (RMID_VAL_ERROR
| RMID_VAL_UNAVAIL
)) {
244 case QOS_L3_OCCUP_EVENT_ID
:
247 case QOS_L3_MBM_TOTAL_EVENT_ID
:
248 m
= &rr
->d
->mbm_total
[rmid
];
250 case QOS_L3_MBM_LOCAL_EVENT_ID
:
251 m
= &rr
->d
->mbm_local
[rmid
];
255 * Code would never reach here because
256 * an invalid event id would fail the __rmid_read.
262 memset(m
, 0, sizeof(struct mbm_state
));
263 m
->prev_bw_msr
= m
->prev_msr
= tval
;
267 chunks
= mbm_overflow_count(m
->prev_msr
, tval
);
271 rr
->val
+= m
->chunks
;
276 * Supporting function to calculate the memory bandwidth
277 * and delta bandwidth in MBps.
279 static void mbm_bw_count(u32 rmid
, struct rmid_read
*rr
)
281 struct rdt_resource
*r
= &rdt_resources_all
[RDT_RESOURCE_L3
];
282 struct mbm_state
*m
= &rr
->d
->mbm_local
[rmid
];
283 u64 tval
, cur_bw
, chunks
;
285 tval
= __rmid_read(rmid
, rr
->evtid
);
286 if (tval
& (RMID_VAL_ERROR
| RMID_VAL_UNAVAIL
))
289 chunks
= mbm_overflow_count(m
->prev_bw_msr
, tval
);
290 m
->chunks_bw
+= chunks
;
291 m
->chunks
= m
->chunks_bw
;
292 cur_bw
= (chunks
* r
->mon_scale
) >> 20;
295 m
->delta_bw
= abs(cur_bw
- m
->prev_bw
);
296 m
->delta_comp
= false;
298 m
->prev_bw_msr
= tval
;
302 * This is called via IPI to read the CQM/MBM counters
305 void mon_event_count(void *info
)
307 struct rdtgroup
*rdtgrp
, *entry
;
308 struct rmid_read
*rr
= info
;
309 struct list_head
*head
;
313 if (__mon_event_count(rdtgrp
->mon
.rmid
, rr
))
317 * For Ctrl groups read data from child monitor groups.
319 head
= &rdtgrp
->mon
.crdtgrp_list
;
321 if (rdtgrp
->type
== RDTCTRL_GROUP
) {
322 list_for_each_entry(entry
, head
, mon
.crdtgrp_list
) {
323 if (__mon_event_count(entry
->mon
.rmid
, rr
))
330 * Feedback loop for MBA software controller (mba_sc)
332 * mba_sc is a feedback loop where we periodically read MBM counters and
333 * adjust the bandwidth percentage values via the IA32_MBA_THRTL_MSRs so
336 * current bandwdith(cur_bw) < user specified bandwidth(user_bw)
338 * This uses the MBM counters to measure the bandwidth and MBA throttle
339 * MSRs to control the bandwidth for a particular rdtgrp. It builds on the
340 * fact that resctrl rdtgroups have both monitoring and control.
342 * The frequency of the checks is 1s and we just tag along the MBM overflow
343 * timer. Having 1s interval makes the calculation of bandwidth simpler.
345 * Although MBA's goal is to restrict the bandwidth to a maximum, there may
346 * be a need to increase the bandwidth to avoid uncecessarily restricting
347 * the L2 <-> L3 traffic.
349 * Since MBA controls the L2 external bandwidth where as MBM measures the
350 * L3 external bandwidth the following sequence could lead to such a
353 * Consider an rdtgroup which had high L3 <-> memory traffic in initial
354 * phases -> mba_sc kicks in and reduced bandwidth percentage values -> but
355 * after some time rdtgroup has mostly L2 <-> L3 traffic.
357 * In this case we may restrict the rdtgroup's L2 <-> L3 traffic as its
358 * throttle MSRs already have low percentage values. To avoid
359 * unnecessarily restricting such rdtgroups, we also increase the bandwidth.
361 static void update_mba_bw(struct rdtgroup
*rgrp
, struct rdt_domain
*dom_mbm
)
363 u32 closid
, rmid
, cur_msr
, cur_msr_val
, new_msr_val
;
364 struct mbm_state
*pmbm_data
, *cmbm_data
;
365 u32 cur_bw
, delta_bw
, user_bw
;
366 struct rdt_resource
*r_mba
;
367 struct rdt_domain
*dom_mba
;
368 struct list_head
*head
;
369 struct rdtgroup
*entry
;
371 if (!is_mbm_local_enabled())
374 r_mba
= &rdt_resources_all
[RDT_RESOURCE_MBA
];
375 closid
= rgrp
->closid
;
376 rmid
= rgrp
->mon
.rmid
;
377 pmbm_data
= &dom_mbm
->mbm_local
[rmid
];
379 dom_mba
= get_domain_from_cpu(smp_processor_id(), r_mba
);
381 pr_warn_once("Failure to get domain for MBA update\n");
385 cur_bw
= pmbm_data
->prev_bw
;
386 user_bw
= dom_mba
->mbps_val
[closid
];
387 delta_bw
= pmbm_data
->delta_bw
;
388 cur_msr_val
= dom_mba
->ctrl_val
[closid
];
391 * For Ctrl groups read data from child monitor groups.
393 head
= &rgrp
->mon
.crdtgrp_list
;
394 list_for_each_entry(entry
, head
, mon
.crdtgrp_list
) {
395 cmbm_data
= &dom_mbm
->mbm_local
[entry
->mon
.rmid
];
396 cur_bw
+= cmbm_data
->prev_bw
;
397 delta_bw
+= cmbm_data
->delta_bw
;
401 * Scale up/down the bandwidth linearly for the ctrl group. The
402 * bandwidth step is the bandwidth granularity specified by the
405 * The delta_bw is used when increasing the bandwidth so that we
406 * dont alternately increase and decrease the control values
409 * For ex: consider cur_bw = 90MBps, user_bw = 100MBps and if
410 * bandwidth step is 20MBps(> user_bw - cur_bw), we would keep
411 * switching between 90 and 110 continuously if we only check
414 if (cur_msr_val
> r_mba
->membw
.min_bw
&& user_bw
< cur_bw
) {
415 new_msr_val
= cur_msr_val
- r_mba
->membw
.bw_gran
;
416 } else if (cur_msr_val
< MAX_MBA_BW
&&
417 (user_bw
> (cur_bw
+ delta_bw
))) {
418 new_msr_val
= cur_msr_val
+ r_mba
->membw
.bw_gran
;
423 cur_msr
= r_mba
->msr_base
+ closid
;
424 wrmsrl(cur_msr
, delay_bw_map(new_msr_val
, r_mba
));
425 dom_mba
->ctrl_val
[closid
] = new_msr_val
;
428 * Delta values are updated dynamically package wise for each
429 * rdtgrp everytime the throttle MSR changes value.
431 * This is because (1)the increase in bandwidth is not perfectly
432 * linear and only "approximately" linear even when the hardware
433 * says it is linear.(2)Also since MBA is a core specific
434 * mechanism, the delta values vary based on number of cores used
437 pmbm_data
->delta_comp
= true;
438 list_for_each_entry(entry
, head
, mon
.crdtgrp_list
) {
439 cmbm_data
= &dom_mbm
->mbm_local
[entry
->mon
.rmid
];
440 cmbm_data
->delta_comp
= true;
444 static void mbm_update(struct rdt_domain
*d
, int rmid
)
452 * This is protected from concurrent reads from user
453 * as both the user and we hold the global mutex.
455 if (is_mbm_total_enabled()) {
456 rr
.evtid
= QOS_L3_MBM_TOTAL_EVENT_ID
;
457 __mon_event_count(rmid
, &rr
);
459 if (is_mbm_local_enabled()) {
460 rr
.evtid
= QOS_L3_MBM_LOCAL_EVENT_ID
;
463 * Call the MBA software controller only for the
464 * control groups and when user has enabled
465 * the software controller explicitly.
467 if (!is_mba_sc(NULL
))
468 __mon_event_count(rmid
, &rr
);
470 mbm_bw_count(rmid
, &rr
);
475 * Handler to scan the limbo list and move the RMIDs
476 * to free list whose occupancy < threshold_occupancy.
478 void cqm_handle_limbo(struct work_struct
*work
)
480 unsigned long delay
= msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL
);
481 int cpu
= smp_processor_id();
482 struct rdt_resource
*r
;
483 struct rdt_domain
*d
;
485 mutex_lock(&rdtgroup_mutex
);
487 r
= &rdt_resources_all
[RDT_RESOURCE_L3
];
488 d
= get_domain_from_cpu(cpu
, r
);
491 pr_warn_once("Failure to get domain for limbo worker\n");
495 __check_limbo(d
, false);
497 if (has_busy_rmid(r
, d
))
498 schedule_delayed_work_on(cpu
, &d
->cqm_limbo
, delay
);
501 mutex_unlock(&rdtgroup_mutex
);
504 void cqm_setup_limbo_handler(struct rdt_domain
*dom
, unsigned long delay_ms
)
506 unsigned long delay
= msecs_to_jiffies(delay_ms
);
509 cpu
= cpumask_any(&dom
->cpu_mask
);
510 dom
->cqm_work_cpu
= cpu
;
512 schedule_delayed_work_on(cpu
, &dom
->cqm_limbo
, delay
);
515 void mbm_handle_overflow(struct work_struct
*work
)
517 unsigned long delay
= msecs_to_jiffies(MBM_OVERFLOW_INTERVAL
);
518 struct rdtgroup
*prgrp
, *crgrp
;
519 int cpu
= smp_processor_id();
520 struct list_head
*head
;
521 struct rdt_domain
*d
;
523 mutex_lock(&rdtgroup_mutex
);
525 if (!static_branch_likely(&rdt_enable_key
))
528 d
= get_domain_from_cpu(cpu
, &rdt_resources_all
[RDT_RESOURCE_L3
]);
532 list_for_each_entry(prgrp
, &rdt_all_groups
, rdtgroup_list
) {
533 mbm_update(d
, prgrp
->mon
.rmid
);
535 head
= &prgrp
->mon
.crdtgrp_list
;
536 list_for_each_entry(crgrp
, head
, mon
.crdtgrp_list
)
537 mbm_update(d
, crgrp
->mon
.rmid
);
540 update_mba_bw(prgrp
, d
);
543 schedule_delayed_work_on(cpu
, &d
->mbm_over
, delay
);
546 mutex_unlock(&rdtgroup_mutex
);
549 void mbm_setup_overflow_handler(struct rdt_domain
*dom
, unsigned long delay_ms
)
551 unsigned long delay
= msecs_to_jiffies(delay_ms
);
554 if (!static_branch_likely(&rdt_enable_key
))
556 cpu
= cpumask_any(&dom
->cpu_mask
);
557 dom
->mbm_work_cpu
= cpu
;
558 schedule_delayed_work_on(cpu
, &dom
->mbm_over
, delay
);
561 static int dom_data_init(struct rdt_resource
*r
)
563 struct rmid_entry
*entry
= NULL
;
566 nr_rmids
= r
->num_rmid
;
567 rmid_ptrs
= kcalloc(nr_rmids
, sizeof(struct rmid_entry
), GFP_KERNEL
);
571 for (i
= 0; i
< nr_rmids
; i
++) {
572 entry
= &rmid_ptrs
[i
];
573 INIT_LIST_HEAD(&entry
->list
);
576 list_add_tail(&entry
->list
, &rmid_free_lru
);
580 * RMID 0 is special and is always allocated. It's used for all
581 * tasks that are not monitored.
583 entry
= __rmid_entry(0);
584 list_del(&entry
->list
);
589 static struct mon_evt llc_occupancy_event
= {
590 .name
= "llc_occupancy",
591 .evtid
= QOS_L3_OCCUP_EVENT_ID
,
594 static struct mon_evt mbm_total_event
= {
595 .name
= "mbm_total_bytes",
596 .evtid
= QOS_L3_MBM_TOTAL_EVENT_ID
,
599 static struct mon_evt mbm_local_event
= {
600 .name
= "mbm_local_bytes",
601 .evtid
= QOS_L3_MBM_LOCAL_EVENT_ID
,
605 * Initialize the event list for the resource.
607 * Note that MBM events are also part of RDT_RESOURCE_L3 resource
608 * because as per the SDM the total and local memory bandwidth
609 * are enumerated as part of L3 monitoring.
611 static void l3_mon_evt_init(struct rdt_resource
*r
)
613 INIT_LIST_HEAD(&r
->evt_list
);
615 if (is_llc_occupancy_enabled())
616 list_add_tail(&llc_occupancy_event
.list
, &r
->evt_list
);
617 if (is_mbm_total_enabled())
618 list_add_tail(&mbm_total_event
.list
, &r
->evt_list
);
619 if (is_mbm_local_enabled())
620 list_add_tail(&mbm_local_event
.list
, &r
->evt_list
);
623 int rdt_get_mon_l3_config(struct rdt_resource
*r
)
625 unsigned int cl_size
= boot_cpu_data
.x86_cache_size
;
628 r
->mon_scale
= boot_cpu_data
.x86_cache_occ_scale
;
629 r
->num_rmid
= boot_cpu_data
.x86_cache_max_rmid
+ 1;
632 * A reasonable upper limit on the max threshold is the number
633 * of lines tagged per RMID if all RMIDs have the same number of
634 * lines tagged in the LLC.
636 * For a 35MB LLC and 56 RMIDs, this is ~1.8% of the LLC.
638 resctrl_cqm_threshold
= cl_size
* 1024 / r
->num_rmid
;
640 /* h/w works in units of "boot_cpu_data.x86_cache_occ_scale" */
641 resctrl_cqm_threshold
/= r
->mon_scale
;
643 ret
= dom_data_init(r
);
649 r
->mon_capable
= true;
650 r
->mon_enabled
= true;