2 * Block rq-qos base io controller
4 * This works similar to wbt with a few exceptions
6 * - It's bio based, so the latency covers the whole block layer in addition to
8 * - We will throttle all IO that comes in here if we need to.
9 * - We use the mean latency over the 100ms window. This is because writes can
10 * be particularly fast, which could give us a false sense of the impact of
11 * other workloads on our protected workload.
12 * - By default there's no throttling, we set the queue_depth to UINT_MAX so
13 * that we can have as many outstanding bio's as we're allowed to. Only at
14 * throttle time do we pay attention to the actual queue depth.
16 * The hierarchy works like the cpu controller does, we track the latency at
17 * every configured node, and each configured node has it's own independent
18 * queue depth. This means that we only care about our latency targets at the
19 * peer level. Some group at the bottom of the hierarchy isn't going to affect
20 * a group at the end of some other path if we're only configred at leaf level.
22 * Consider the following
26 * fast (target=5ms) slow (target=10ms)
28 * a b normal(15ms) unloved
30 * "a" and "b" have no target, but their combined io under "fast" cannot exceed
31 * an average latency of 5ms. If it does then we will throttle the "slow"
32 * group. In the case of "normal", if it exceeds its 15ms target, we will
33 * throttle "unloved", but nobody else.
35 * In this example "fast", "slow", and "normal" will be the only groups actually
36 * accounting their io latencies. We have to walk up the heirarchy to the root
37 * on every submit and complete so we can do the appropriate stat recording and
38 * adjust the queue depth of ourselves if needed.
40 * There are 2 ways we throttle IO.
42 * 1) Queue depth throttling. As we throttle down we will adjust the maximum
43 * number of IO's we're allowed to have in flight. This starts at (u64)-1 down
44 * to 1. If the group is only ever submitting IO for itself then this is the
45 * only way we throttle.
47 * 2) Induced delay throttling. This is for the case that a group is generating
48 * IO that has to be issued by the root cg to avoid priority inversion. So think
49 * REQ_META or REQ_SWAP. If we are already at qd == 1 and we're getting a lot
50 * of work done for us on behalf of the root cg and are being asked to scale
51 * down more then we induce a latency at userspace return. We accumulate the
52 * total amount of time we need to be punished by doing
54 * total_time += min_lat_nsec - actual_io_completion
56 * and then at throttle time will do
58 * throttle_time = min(total_time, NSEC_PER_SEC)
60 * This induced delay will throttle back the activity that is generating the
61 * root cg issued io's, wethere that's some metadata intensive operation or the
62 * group is using so much memory that it is pushing us into swap.
64 * Copyright (C) 2018 Josef Bacik
66 #include <linux/kernel.h>
67 #include <linux/blk_types.h>
68 #include <linux/backing-dev.h>
69 #include <linux/module.h>
70 #include <linux/timer.h>
71 #include <linux/memcontrol.h>
72 #include <linux/sched/loadavg.h>
73 #include <linux/sched/signal.h>
74 #include <trace/events/block.h>
75 #include "blk-rq-qos.h"
78 #define DEFAULT_SCALE_COOKIE 1000000U
80 static struct blkcg_policy blkcg_policy_iolatency
;
83 struct blk_iolatency
{
85 struct timer_list timer
;
89 static inline struct blk_iolatency
*BLKIOLATENCY(struct rq_qos
*rqos
)
91 return container_of(rqos
, struct blk_iolatency
, rqos
);
94 static inline bool blk_iolatency_enabled(struct blk_iolatency
*blkiolat
)
96 return atomic_read(&blkiolat
->enabled
) > 0;
99 struct child_latency_info
{
102 /* Last time we adjusted the scale of everybody. */
103 u64 last_scale_event
;
105 /* The latency that we missed. */
108 /* Total io's from all of our children for the last summation. */
111 /* The guy who actually changed the latency numbers. */
112 struct iolatency_grp
*scale_grp
;
114 /* Cookie to tell if we need to scale up or down. */
115 atomic_t scale_cookie
;
118 struct percentile_stats
{
123 struct latency_stat
{
125 struct percentile_stats ps
;
126 struct blk_rq_stat rqs
;
130 struct iolatency_grp
{
131 struct blkg_policy_data pd
;
132 struct latency_stat __percpu
*stats
;
133 struct latency_stat cur_stat
;
134 struct blk_iolatency
*blkiolat
;
135 struct rq_depth rq_depth
;
136 struct rq_wait rq_wait
;
137 atomic64_t window_start
;
138 atomic_t scale_cookie
;
142 /* total running average of our io latency. */
145 /* Our current number of IO's for the last summation. */
149 struct child_latency_info child_lat
;
152 #define BLKIOLATENCY_MIN_WIN_SIZE (100 * NSEC_PER_MSEC)
153 #define BLKIOLATENCY_MAX_WIN_SIZE NSEC_PER_SEC
155 * These are the constants used to fake the fixed-point moving average
156 * calculation just like load average. The call to calc_load() folds
157 * (FIXED_1 (2048) - exp_factor) * new_sample into lat_avg. The sampling
158 * window size is bucketed to try to approximately calculate average
159 * latency such that 1/exp (decay rate) is [1 min, 2.5 min) when windows
160 * elapse immediately. Note, windows only elapse with IO activity. Idle
161 * periods extend the most recent window.
163 #define BLKIOLATENCY_NR_EXP_FACTORS 5
164 #define BLKIOLATENCY_EXP_BUCKET_SIZE (BLKIOLATENCY_MAX_WIN_SIZE / \
165 (BLKIOLATENCY_NR_EXP_FACTORS - 1))
166 static const u64 iolatency_exp_factors
[BLKIOLATENCY_NR_EXP_FACTORS
] = {
167 2045, // exp(1/600) - 600 samples
168 2039, // exp(1/240) - 240 samples
169 2031, // exp(1/120) - 120 samples
170 2023, // exp(1/80) - 80 samples
171 2014, // exp(1/60) - 60 samples
174 static inline struct iolatency_grp
*pd_to_lat(struct blkg_policy_data
*pd
)
176 return pd
? container_of(pd
, struct iolatency_grp
, pd
) : NULL
;
179 static inline struct iolatency_grp
*blkg_to_lat(struct blkcg_gq
*blkg
)
181 return pd_to_lat(blkg_to_pd(blkg
, &blkcg_policy_iolatency
));
184 static inline struct blkcg_gq
*lat_to_blkg(struct iolatency_grp
*iolat
)
186 return pd_to_blkg(&iolat
->pd
);
189 static inline void latency_stat_init(struct iolatency_grp
*iolat
,
190 struct latency_stat
*stat
)
196 blk_rq_stat_init(&stat
->rqs
);
199 static inline void latency_stat_sum(struct iolatency_grp
*iolat
,
200 struct latency_stat
*sum
,
201 struct latency_stat
*stat
)
204 sum
->ps
.total
+= stat
->ps
.total
;
205 sum
->ps
.missed
+= stat
->ps
.missed
;
207 blk_rq_stat_sum(&sum
->rqs
, &stat
->rqs
);
210 static inline void latency_stat_record_time(struct iolatency_grp
*iolat
,
213 struct latency_stat
*stat
= get_cpu_ptr(iolat
->stats
);
215 if (req_time
>= iolat
->min_lat_nsec
)
219 blk_rq_stat_add(&stat
->rqs
, req_time
);
223 static inline bool latency_sum_ok(struct iolatency_grp
*iolat
,
224 struct latency_stat
*stat
)
227 u64 thresh
= div64_u64(stat
->ps
.total
, 10);
228 thresh
= max(thresh
, 1ULL);
229 return stat
->ps
.missed
< thresh
;
231 return stat
->rqs
.mean
<= iolat
->min_lat_nsec
;
234 static inline u64
latency_stat_samples(struct iolatency_grp
*iolat
,
235 struct latency_stat
*stat
)
238 return stat
->ps
.total
;
239 return stat
->rqs
.nr_samples
;
242 static inline void iolat_update_total_lat_avg(struct iolatency_grp
*iolat
,
243 struct latency_stat
*stat
)
251 * calc_load() takes in a number stored in fixed point representation.
252 * Because we are using this for IO time in ns, the values stored
253 * are significantly larger than the FIXED_1 denominator (2048).
254 * Therefore, rounding errors in the calculation are negligible and
257 exp_idx
= min_t(int, BLKIOLATENCY_NR_EXP_FACTORS
- 1,
258 div64_u64(iolat
->cur_win_nsec
,
259 BLKIOLATENCY_EXP_BUCKET_SIZE
));
260 iolat
->lat_avg
= calc_load(iolat
->lat_avg
,
261 iolatency_exp_factors
[exp_idx
],
265 static void iolat_cleanup_cb(struct rq_wait
*rqw
, void *private_data
)
267 atomic_dec(&rqw
->inflight
);
271 static bool iolat_acquire_inflight(struct rq_wait
*rqw
, void *private_data
)
273 struct iolatency_grp
*iolat
= private_data
;
274 return rq_wait_inc_below(rqw
, iolat
->rq_depth
.max_depth
);
277 static void __blkcg_iolatency_throttle(struct rq_qos
*rqos
,
278 struct iolatency_grp
*iolat
,
282 struct rq_wait
*rqw
= &iolat
->rq_wait
;
283 unsigned use_delay
= atomic_read(&lat_to_blkg(iolat
)->use_delay
);
286 blkcg_schedule_throttle(rqos
->q
, use_memdelay
);
289 * To avoid priority inversions we want to just take a slot if we are
290 * issuing as root. If we're being killed off there's no point in
291 * delaying things, we may have been killed by OOM so throttling may
292 * make recovery take even longer, so just let the IO's through so the
295 if (issue_as_root
|| fatal_signal_pending(current
)) {
296 atomic_inc(&rqw
->inflight
);
300 rq_qos_wait(rqw
, iolat
, iolat_acquire_inflight
, iolat_cleanup_cb
);
303 #define SCALE_DOWN_FACTOR 2
304 #define SCALE_UP_FACTOR 4
306 static inline unsigned long scale_amount(unsigned long qd
, bool up
)
308 return max(up
? qd
>> SCALE_UP_FACTOR
: qd
>> SCALE_DOWN_FACTOR
, 1UL);
312 * We scale the qd down faster than we scale up, so we need to use this helper
313 * to adjust the scale_cookie accordingly so we don't prematurely get
314 * scale_cookie at DEFAULT_SCALE_COOKIE and unthrottle too much.
316 * Each group has their own local copy of the last scale cookie they saw, so if
317 * the global scale cookie goes up or down they know which way they need to go
318 * based on their last knowledge of it.
320 static void scale_cookie_change(struct blk_iolatency
*blkiolat
,
321 struct child_latency_info
*lat_info
,
324 unsigned long qd
= blkiolat
->rqos
.q
->nr_requests
;
325 unsigned long scale
= scale_amount(qd
, up
);
326 unsigned long old
= atomic_read(&lat_info
->scale_cookie
);
327 unsigned long max_scale
= qd
<< 1;
328 unsigned long diff
= 0;
330 if (old
< DEFAULT_SCALE_COOKIE
)
331 diff
= DEFAULT_SCALE_COOKIE
- old
;
334 if (scale
+ old
> DEFAULT_SCALE_COOKIE
)
335 atomic_set(&lat_info
->scale_cookie
,
336 DEFAULT_SCALE_COOKIE
);
338 atomic_inc(&lat_info
->scale_cookie
);
340 atomic_add(scale
, &lat_info
->scale_cookie
);
343 * We don't want to dig a hole so deep that it takes us hours to
344 * dig out of it. Just enough that we don't throttle/unthrottle
345 * with jagged workloads but can still unthrottle once pressure
346 * has sufficiently dissipated.
349 if (diff
< max_scale
)
350 atomic_dec(&lat_info
->scale_cookie
);
352 atomic_sub(scale
, &lat_info
->scale_cookie
);
358 * Change the queue depth of the iolatency_grp. We add/subtract 1/16th of the
359 * queue depth at a time so we don't get wild swings and hopefully dial in to
360 * fairer distribution of the overall queue depth.
362 static void scale_change(struct iolatency_grp
*iolat
, bool up
)
364 unsigned long qd
= iolat
->blkiolat
->rqos
.q
->nr_requests
;
365 unsigned long scale
= scale_amount(qd
, up
);
366 unsigned long old
= iolat
->rq_depth
.max_depth
;
372 if (old
== 1 && blkcg_unuse_delay(lat_to_blkg(iolat
)))
378 iolat
->rq_depth
.max_depth
= old
;
379 wake_up_all(&iolat
->rq_wait
.wait
);
383 iolat
->rq_depth
.max_depth
= max(old
, 1UL);
387 /* Check our parent and see if the scale cookie has changed. */
388 static void check_scale_change(struct iolatency_grp
*iolat
)
390 struct iolatency_grp
*parent
;
391 struct child_latency_info
*lat_info
;
392 unsigned int cur_cookie
;
393 unsigned int our_cookie
= atomic_read(&iolat
->scale_cookie
);
398 if (lat_to_blkg(iolat
)->parent
== NULL
)
401 parent
= blkg_to_lat(lat_to_blkg(iolat
)->parent
);
405 lat_info
= &parent
->child_lat
;
406 cur_cookie
= atomic_read(&lat_info
->scale_cookie
);
407 scale_lat
= READ_ONCE(lat_info
->scale_lat
);
409 if (cur_cookie
< our_cookie
)
411 else if (cur_cookie
> our_cookie
)
416 old
= atomic_cmpxchg(&iolat
->scale_cookie
, our_cookie
, cur_cookie
);
418 /* Somebody beat us to the punch, just bail. */
419 if (old
!= our_cookie
)
422 if (direction
< 0 && iolat
->min_lat_nsec
) {
425 if (!scale_lat
|| iolat
->min_lat_nsec
<= scale_lat
)
429 * Sometimes high priority groups are their own worst enemy, so
430 * instead of taking it out on some poor other group that did 5%
431 * or less of the IO's for the last summation just skip this
434 samples_thresh
= lat_info
->nr_samples
* 5;
435 samples_thresh
= max(1ULL, div64_u64(samples_thresh
, 100));
436 if (iolat
->nr_samples
<= samples_thresh
)
440 /* We're as low as we can go. */
441 if (iolat
->rq_depth
.max_depth
== 1 && direction
< 0) {
442 blkcg_use_delay(lat_to_blkg(iolat
));
446 /* We're back to the default cookie, unthrottle all the things. */
447 if (cur_cookie
== DEFAULT_SCALE_COOKIE
) {
448 blkcg_clear_delay(lat_to_blkg(iolat
));
449 iolat
->rq_depth
.max_depth
= UINT_MAX
;
450 wake_up_all(&iolat
->rq_wait
.wait
);
454 scale_change(iolat
, direction
> 0);
457 static void blkcg_iolatency_throttle(struct rq_qos
*rqos
, struct bio
*bio
)
459 struct blk_iolatency
*blkiolat
= BLKIOLATENCY(rqos
);
460 struct blkcg_gq
*blkg
= bio
->bi_blkg
;
461 bool issue_as_root
= bio_issue_as_root_blkg(bio
);
463 if (!blk_iolatency_enabled(blkiolat
))
466 while (blkg
&& blkg
->parent
) {
467 struct iolatency_grp
*iolat
= blkg_to_lat(blkg
);
473 check_scale_change(iolat
);
474 __blkcg_iolatency_throttle(rqos
, iolat
, issue_as_root
,
475 (bio
->bi_opf
& REQ_SWAP
) == REQ_SWAP
);
478 if (!timer_pending(&blkiolat
->timer
))
479 mod_timer(&blkiolat
->timer
, jiffies
+ HZ
);
482 static void iolatency_record_time(struct iolatency_grp
*iolat
,
483 struct bio_issue
*issue
, u64 now
,
486 u64 start
= bio_issue_time(issue
);
490 * Have to do this so we are truncated to the correct time that our
491 * issue is truncated to.
493 now
= __bio_issue_time(now
);
498 req_time
= now
- start
;
501 * We don't want to count issue_as_root bio's in the cgroups latency
502 * statistics as it could skew the numbers downwards.
504 if (unlikely(issue_as_root
&& iolat
->rq_depth
.max_depth
!= UINT_MAX
)) {
505 u64 sub
= iolat
->min_lat_nsec
;
507 blkcg_add_delay(lat_to_blkg(iolat
), now
, sub
- req_time
);
511 latency_stat_record_time(iolat
, req_time
);
514 #define BLKIOLATENCY_MIN_ADJUST_TIME (500 * NSEC_PER_MSEC)
515 #define BLKIOLATENCY_MIN_GOOD_SAMPLES 5
517 static void iolatency_check_latencies(struct iolatency_grp
*iolat
, u64 now
)
519 struct blkcg_gq
*blkg
= lat_to_blkg(iolat
);
520 struct iolatency_grp
*parent
;
521 struct child_latency_info
*lat_info
;
522 struct latency_stat stat
;
526 latency_stat_init(iolat
, &stat
);
528 for_each_online_cpu(cpu
) {
529 struct latency_stat
*s
;
530 s
= per_cpu_ptr(iolat
->stats
, cpu
);
531 latency_stat_sum(iolat
, &stat
, s
);
532 latency_stat_init(iolat
, s
);
536 parent
= blkg_to_lat(blkg
->parent
);
540 lat_info
= &parent
->child_lat
;
542 iolat_update_total_lat_avg(iolat
, &stat
);
544 /* Everything is ok and we don't need to adjust the scale. */
545 if (latency_sum_ok(iolat
, &stat
) &&
546 atomic_read(&lat_info
->scale_cookie
) == DEFAULT_SCALE_COOKIE
)
549 /* Somebody beat us to the punch, just bail. */
550 spin_lock_irqsave(&lat_info
->lock
, flags
);
552 latency_stat_sum(iolat
, &iolat
->cur_stat
, &stat
);
553 lat_info
->nr_samples
-= iolat
->nr_samples
;
554 lat_info
->nr_samples
+= latency_stat_samples(iolat
, &iolat
->cur_stat
);
555 iolat
->nr_samples
= latency_stat_samples(iolat
, &iolat
->cur_stat
);
557 if ((lat_info
->last_scale_event
>= now
||
558 now
- lat_info
->last_scale_event
< BLKIOLATENCY_MIN_ADJUST_TIME
))
561 if (latency_sum_ok(iolat
, &iolat
->cur_stat
) &&
562 latency_sum_ok(iolat
, &stat
)) {
563 if (latency_stat_samples(iolat
, &iolat
->cur_stat
) <
564 BLKIOLATENCY_MIN_GOOD_SAMPLES
)
566 if (lat_info
->scale_grp
== iolat
) {
567 lat_info
->last_scale_event
= now
;
568 scale_cookie_change(iolat
->blkiolat
, lat_info
, true);
570 } else if (lat_info
->scale_lat
== 0 ||
571 lat_info
->scale_lat
>= iolat
->min_lat_nsec
) {
572 lat_info
->last_scale_event
= now
;
573 if (!lat_info
->scale_grp
||
574 lat_info
->scale_lat
> iolat
->min_lat_nsec
) {
575 WRITE_ONCE(lat_info
->scale_lat
, iolat
->min_lat_nsec
);
576 lat_info
->scale_grp
= iolat
;
578 scale_cookie_change(iolat
->blkiolat
, lat_info
, false);
580 latency_stat_init(iolat
, &iolat
->cur_stat
);
582 spin_unlock_irqrestore(&lat_info
->lock
, flags
);
585 static void blkcg_iolatency_done_bio(struct rq_qos
*rqos
, struct bio
*bio
)
587 struct blkcg_gq
*blkg
;
589 struct iolatency_grp
*iolat
;
591 u64 now
= ktime_to_ns(ktime_get());
592 bool issue_as_root
= bio_issue_as_root_blkg(bio
);
593 bool enabled
= false;
596 if (!blkg
|| !bio_flagged(bio
, BIO_TRACKED
))
599 iolat
= blkg_to_lat(bio
->bi_blkg
);
603 enabled
= blk_iolatency_enabled(iolat
->blkiolat
);
604 while (blkg
&& blkg
->parent
) {
605 iolat
= blkg_to_lat(blkg
);
610 rqw
= &iolat
->rq_wait
;
612 atomic_dec(&rqw
->inflight
);
613 if (!enabled
|| iolat
->min_lat_nsec
== 0)
615 iolatency_record_time(iolat
, &bio
->bi_issue
, now
,
617 window_start
= atomic64_read(&iolat
->window_start
);
618 if (now
> window_start
&&
619 (now
- window_start
) >= iolat
->cur_win_nsec
) {
620 if (atomic64_cmpxchg(&iolat
->window_start
,
621 window_start
, now
) == window_start
)
622 iolatency_check_latencies(iolat
, now
);
630 static void blkcg_iolatency_cleanup(struct rq_qos
*rqos
, struct bio
*bio
)
632 struct blkcg_gq
*blkg
;
635 while (blkg
&& blkg
->parent
) {
637 struct iolatency_grp
*iolat
;
639 iolat
= blkg_to_lat(blkg
);
643 rqw
= &iolat
->rq_wait
;
644 atomic_dec(&rqw
->inflight
);
651 static void blkcg_iolatency_exit(struct rq_qos
*rqos
)
653 struct blk_iolatency
*blkiolat
= BLKIOLATENCY(rqos
);
655 del_timer_sync(&blkiolat
->timer
);
656 blkcg_deactivate_policy(rqos
->q
, &blkcg_policy_iolatency
);
660 static struct rq_qos_ops blkcg_iolatency_ops
= {
661 .throttle
= blkcg_iolatency_throttle
,
662 .cleanup
= blkcg_iolatency_cleanup
,
663 .done_bio
= blkcg_iolatency_done_bio
,
664 .exit
= blkcg_iolatency_exit
,
667 static void blkiolatency_timer_fn(struct timer_list
*t
)
669 struct blk_iolatency
*blkiolat
= from_timer(blkiolat
, t
, timer
);
670 struct blkcg_gq
*blkg
;
671 struct cgroup_subsys_state
*pos_css
;
672 u64 now
= ktime_to_ns(ktime_get());
675 blkg_for_each_descendant_pre(blkg
, pos_css
,
676 blkiolat
->rqos
.q
->root_blkg
) {
677 struct iolatency_grp
*iolat
;
678 struct child_latency_info
*lat_info
;
683 * We could be exiting, don't access the pd unless we have a
686 if (!blkg_tryget(blkg
))
689 iolat
= blkg_to_lat(blkg
);
693 lat_info
= &iolat
->child_lat
;
694 cookie
= atomic_read(&lat_info
->scale_cookie
);
696 if (cookie
>= DEFAULT_SCALE_COOKIE
)
699 spin_lock_irqsave(&lat_info
->lock
, flags
);
700 if (lat_info
->last_scale_event
>= now
)
704 * We scaled down but don't have a scale_grp, scale up and carry
707 if (lat_info
->scale_grp
== NULL
) {
708 scale_cookie_change(iolat
->blkiolat
, lat_info
, true);
713 * It's been 5 seconds since our last scale event, clear the
714 * scale grp in case the group that needed the scale down isn't
715 * doing any IO currently.
717 if (now
- lat_info
->last_scale_event
>=
718 ((u64
)NSEC_PER_SEC
* 5))
719 lat_info
->scale_grp
= NULL
;
721 spin_unlock_irqrestore(&lat_info
->lock
, flags
);
728 int blk_iolatency_init(struct request_queue
*q
)
730 struct blk_iolatency
*blkiolat
;
734 blkiolat
= kzalloc(sizeof(*blkiolat
), GFP_KERNEL
);
738 rqos
= &blkiolat
->rqos
;
739 rqos
->id
= RQ_QOS_CGROUP
;
740 rqos
->ops
= &blkcg_iolatency_ops
;
745 ret
= blkcg_activate_policy(q
, &blkcg_policy_iolatency
);
752 timer_setup(&blkiolat
->timer
, blkiolatency_timer_fn
, 0);
757 static void iolatency_set_min_lat_nsec(struct blkcg_gq
*blkg
, u64 val
)
759 struct iolatency_grp
*iolat
= blkg_to_lat(blkg
);
760 struct blk_iolatency
*blkiolat
= iolat
->blkiolat
;
761 u64 oldval
= iolat
->min_lat_nsec
;
763 iolat
->min_lat_nsec
= val
;
764 iolat
->cur_win_nsec
= max_t(u64
, val
<< 4, BLKIOLATENCY_MIN_WIN_SIZE
);
765 iolat
->cur_win_nsec
= min_t(u64
, iolat
->cur_win_nsec
,
766 BLKIOLATENCY_MAX_WIN_SIZE
);
769 atomic_inc(&blkiolat
->enabled
);
771 atomic_dec(&blkiolat
->enabled
);
774 static void iolatency_clear_scaling(struct blkcg_gq
*blkg
)
777 struct iolatency_grp
*iolat
= blkg_to_lat(blkg
->parent
);
778 struct child_latency_info
*lat_info
;
782 lat_info
= &iolat
->child_lat
;
783 spin_lock(&lat_info
->lock
);
784 atomic_set(&lat_info
->scale_cookie
, DEFAULT_SCALE_COOKIE
);
785 lat_info
->last_scale_event
= 0;
786 lat_info
->scale_grp
= NULL
;
787 lat_info
->scale_lat
= 0;
788 spin_unlock(&lat_info
->lock
);
792 static ssize_t
iolatency_set_limit(struct kernfs_open_file
*of
, char *buf
,
793 size_t nbytes
, loff_t off
)
795 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
796 struct blkcg_gq
*blkg
;
797 struct blkg_conf_ctx ctx
;
798 struct iolatency_grp
*iolat
;
804 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_iolatency
, buf
, &ctx
);
808 iolat
= blkg_to_lat(ctx
.blkg
);
812 while ((tok
= strsep(&p
, " "))) {
814 char val
[21]; /* 18446744073709551616 */
816 if (sscanf(tok
, "%15[^=]=%20s", key
, val
) != 2)
819 if (!strcmp(key
, "target")) {
822 if (!strcmp(val
, "max"))
824 else if (sscanf(val
, "%llu", &v
) == 1)
825 lat_val
= v
* NSEC_PER_USEC
;
833 /* Walk up the tree to see if our new val is lower than it should be. */
835 oldval
= iolat
->min_lat_nsec
;
837 iolatency_set_min_lat_nsec(blkg
, lat_val
);
838 if (oldval
!= iolat
->min_lat_nsec
) {
839 iolatency_clear_scaling(blkg
);
844 blkg_conf_finish(&ctx
);
845 return ret
?: nbytes
;
848 static u64
iolatency_prfill_limit(struct seq_file
*sf
,
849 struct blkg_policy_data
*pd
, int off
)
851 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
852 const char *dname
= blkg_dev_name(pd
->blkg
);
854 if (!dname
|| !iolat
->min_lat_nsec
)
856 seq_printf(sf
, "%s target=%llu\n",
857 dname
, div_u64(iolat
->min_lat_nsec
, NSEC_PER_USEC
));
861 static int iolatency_print_limit(struct seq_file
*sf
, void *v
)
863 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
864 iolatency_prfill_limit
,
865 &blkcg_policy_iolatency
, seq_cft(sf
)->private, false);
869 static size_t iolatency_ssd_stat(struct iolatency_grp
*iolat
, char *buf
,
872 struct latency_stat stat
;
875 latency_stat_init(iolat
, &stat
);
877 for_each_online_cpu(cpu
) {
878 struct latency_stat
*s
;
879 s
= per_cpu_ptr(iolat
->stats
, cpu
);
880 latency_stat_sum(iolat
, &stat
, s
);
884 if (iolat
->rq_depth
.max_depth
== UINT_MAX
)
885 return scnprintf(buf
, size
, " missed=%llu total=%llu depth=max",
886 (unsigned long long)stat
.ps
.missed
,
887 (unsigned long long)stat
.ps
.total
);
888 return scnprintf(buf
, size
, " missed=%llu total=%llu depth=%u",
889 (unsigned long long)stat
.ps
.missed
,
890 (unsigned long long)stat
.ps
.total
,
891 iolat
->rq_depth
.max_depth
);
894 static size_t iolatency_pd_stat(struct blkg_policy_data
*pd
, char *buf
,
897 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
898 unsigned long long avg_lat
;
899 unsigned long long cur_win
;
902 return iolatency_ssd_stat(iolat
, buf
, size
);
904 avg_lat
= div64_u64(iolat
->lat_avg
, NSEC_PER_USEC
);
905 cur_win
= div64_u64(iolat
->cur_win_nsec
, NSEC_PER_MSEC
);
906 if (iolat
->rq_depth
.max_depth
== UINT_MAX
)
907 return scnprintf(buf
, size
, " depth=max avg_lat=%llu win=%llu",
910 return scnprintf(buf
, size
, " depth=%u avg_lat=%llu win=%llu",
911 iolat
->rq_depth
.max_depth
, avg_lat
, cur_win
);
915 static struct blkg_policy_data
*iolatency_pd_alloc(gfp_t gfp
, int node
)
917 struct iolatency_grp
*iolat
;
919 iolat
= kzalloc_node(sizeof(*iolat
), gfp
, node
);
922 iolat
->stats
= __alloc_percpu_gfp(sizeof(struct latency_stat
),
923 __alignof__(struct latency_stat
), gfp
);
931 static void iolatency_pd_init(struct blkg_policy_data
*pd
)
933 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
934 struct blkcg_gq
*blkg
= lat_to_blkg(iolat
);
935 struct rq_qos
*rqos
= blkcg_rq_qos(blkg
->q
);
936 struct blk_iolatency
*blkiolat
= BLKIOLATENCY(rqos
);
937 u64 now
= ktime_to_ns(ktime_get());
940 if (blk_queue_nonrot(blkg
->q
))
945 for_each_possible_cpu(cpu
) {
946 struct latency_stat
*stat
;
947 stat
= per_cpu_ptr(iolat
->stats
, cpu
);
948 latency_stat_init(iolat
, stat
);
951 latency_stat_init(iolat
, &iolat
->cur_stat
);
952 rq_wait_init(&iolat
->rq_wait
);
953 spin_lock_init(&iolat
->child_lat
.lock
);
954 iolat
->rq_depth
.queue_depth
= blkg
->q
->nr_requests
;
955 iolat
->rq_depth
.max_depth
= UINT_MAX
;
956 iolat
->rq_depth
.default_depth
= iolat
->rq_depth
.queue_depth
;
957 iolat
->blkiolat
= blkiolat
;
958 iolat
->cur_win_nsec
= 100 * NSEC_PER_MSEC
;
959 atomic64_set(&iolat
->window_start
, now
);
962 * We init things in list order, so the pd for the parent may not be
963 * init'ed yet for whatever reason.
965 if (blkg
->parent
&& blkg_to_pd(blkg
->parent
, &blkcg_policy_iolatency
)) {
966 struct iolatency_grp
*parent
= blkg_to_lat(blkg
->parent
);
967 atomic_set(&iolat
->scale_cookie
,
968 atomic_read(&parent
->child_lat
.scale_cookie
));
970 atomic_set(&iolat
->scale_cookie
, DEFAULT_SCALE_COOKIE
);
973 atomic_set(&iolat
->child_lat
.scale_cookie
, DEFAULT_SCALE_COOKIE
);
976 static void iolatency_pd_offline(struct blkg_policy_data
*pd
)
978 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
979 struct blkcg_gq
*blkg
= lat_to_blkg(iolat
);
981 iolatency_set_min_lat_nsec(blkg
, 0);
982 iolatency_clear_scaling(blkg
);
985 static void iolatency_pd_free(struct blkg_policy_data
*pd
)
987 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
988 free_percpu(iolat
->stats
);
992 static struct cftype iolatency_files
[] = {
995 .flags
= CFTYPE_NOT_ON_ROOT
,
996 .seq_show
= iolatency_print_limit
,
997 .write
= iolatency_set_limit
,
1002 static struct blkcg_policy blkcg_policy_iolatency
= {
1003 .dfl_cftypes
= iolatency_files
,
1004 .pd_alloc_fn
= iolatency_pd_alloc
,
1005 .pd_init_fn
= iolatency_pd_init
,
1006 .pd_offline_fn
= iolatency_pd_offline
,
1007 .pd_free_fn
= iolatency_pd_free
,
1008 .pd_stat_fn
= iolatency_pd_stat
,
1011 static int __init
iolatency_init(void)
1013 return blkcg_policy_register(&blkcg_policy_iolatency
);
1016 static void __exit
iolatency_exit(void)
1018 return blkcg_policy_unregister(&blkcg_policy_iolatency
);
1021 module_init(iolatency_init
);
1022 module_exit(iolatency_exit
);