1 // SPDX-License-Identifier: GPL-2.0
3 * Block rq-qos base io controller
5 * This works similar to wbt with a few exceptions
7 * - It's bio based, so the latency covers the whole block layer in addition to
9 * - We will throttle all IO that comes in here if we need to.
10 * - We use the mean latency over the 100ms window. This is because writes can
11 * be particularly fast, which could give us a false sense of the impact of
12 * other workloads on our protected workload.
13 * - By default there's no throttling, we set the queue_depth to UINT_MAX so
14 * that we can have as many outstanding bio's as we're allowed to. Only at
15 * throttle time do we pay attention to the actual queue depth.
17 * The hierarchy works like the cpu controller does, we track the latency at
18 * every configured node, and each configured node has it's own independent
19 * queue depth. This means that we only care about our latency targets at the
20 * peer level. Some group at the bottom of the hierarchy isn't going to affect
21 * a group at the end of some other path if we're only configred at leaf level.
23 * Consider the following
27 * fast (target=5ms) slow (target=10ms)
29 * a b normal(15ms) unloved
31 * "a" and "b" have no target, but their combined io under "fast" cannot exceed
32 * an average latency of 5ms. If it does then we will throttle the "slow"
33 * group. In the case of "normal", if it exceeds its 15ms target, we will
34 * throttle "unloved", but nobody else.
36 * In this example "fast", "slow", and "normal" will be the only groups actually
37 * accounting their io latencies. We have to walk up the heirarchy to the root
38 * on every submit and complete so we can do the appropriate stat recording and
39 * adjust the queue depth of ourselves if needed.
41 * There are 2 ways we throttle IO.
43 * 1) Queue depth throttling. As we throttle down we will adjust the maximum
44 * number of IO's we're allowed to have in flight. This starts at (u64)-1 down
45 * to 1. If the group is only ever submitting IO for itself then this is the
46 * only way we throttle.
48 * 2) Induced delay throttling. This is for the case that a group is generating
49 * IO that has to be issued by the root cg to avoid priority inversion. So think
50 * REQ_META or REQ_SWAP. If we are already at qd == 1 and we're getting a lot
51 * of work done for us on behalf of the root cg and are being asked to scale
52 * down more then we induce a latency at userspace return. We accumulate the
53 * total amount of time we need to be punished by doing
55 * total_time += min_lat_nsec - actual_io_completion
57 * and then at throttle time will do
59 * throttle_time = min(total_time, NSEC_PER_SEC)
61 * This induced delay will throttle back the activity that is generating the
62 * root cg issued io's, wethere that's some metadata intensive operation or the
63 * group is using so much memory that it is pushing us into swap.
65 * Copyright (C) 2018 Josef Bacik
67 #include <linux/kernel.h>
68 #include <linux/blk_types.h>
69 #include <linux/backing-dev.h>
70 #include <linux/module.h>
71 #include <linux/timer.h>
72 #include <linux/memcontrol.h>
73 #include <linux/sched/loadavg.h>
74 #include <linux/sched/signal.h>
75 #include <trace/events/block.h>
76 #include <linux/blk-mq.h>
77 #include "blk-rq-qos.h"
79 #include "blk-cgroup.h"
82 #define DEFAULT_SCALE_COOKIE 1000000U
84 static struct blkcg_policy blkcg_policy_iolatency
;
87 struct blk_iolatency
{
89 struct timer_list timer
;
92 * ->enabled is the master enable switch gating the throttling logic and
93 * inflight tracking. The number of cgroups which have iolat enabled is
94 * tracked in ->enable_cnt, and ->enable is flipped on/off accordingly
95 * from ->enable_work with the request_queue frozen. For details, See
96 * blkiolatency_enable_work_fn().
100 struct work_struct enable_work
;
103 static inline struct blk_iolatency
*BLKIOLATENCY(struct rq_qos
*rqos
)
105 return container_of(rqos
, struct blk_iolatency
, rqos
);
108 struct child_latency_info
{
111 /* Last time we adjusted the scale of everybody. */
112 u64 last_scale_event
;
114 /* The latency that we missed. */
117 /* Total io's from all of our children for the last summation. */
120 /* The guy who actually changed the latency numbers. */
121 struct iolatency_grp
*scale_grp
;
123 /* Cookie to tell if we need to scale up or down. */
124 atomic_t scale_cookie
;
127 struct percentile_stats
{
132 struct latency_stat
{
134 struct percentile_stats ps
;
135 struct blk_rq_stat rqs
;
139 struct iolatency_grp
{
140 struct blkg_policy_data pd
;
141 struct latency_stat __percpu
*stats
;
142 struct latency_stat cur_stat
;
143 struct blk_iolatency
*blkiolat
;
144 struct rq_depth rq_depth
;
145 struct rq_wait rq_wait
;
146 atomic64_t window_start
;
147 atomic_t scale_cookie
;
151 /* total running average of our io latency. */
154 /* Our current number of IO's for the last summation. */
158 struct child_latency_info child_lat
;
161 #define BLKIOLATENCY_MIN_WIN_SIZE (100 * NSEC_PER_MSEC)
162 #define BLKIOLATENCY_MAX_WIN_SIZE NSEC_PER_SEC
164 * These are the constants used to fake the fixed-point moving average
165 * calculation just like load average. The call to calc_load() folds
166 * (FIXED_1 (2048) - exp_factor) * new_sample into lat_avg. The sampling
167 * window size is bucketed to try to approximately calculate average
168 * latency such that 1/exp (decay rate) is [1 min, 2.5 min) when windows
169 * elapse immediately. Note, windows only elapse with IO activity. Idle
170 * periods extend the most recent window.
172 #define BLKIOLATENCY_NR_EXP_FACTORS 5
173 #define BLKIOLATENCY_EXP_BUCKET_SIZE (BLKIOLATENCY_MAX_WIN_SIZE / \
174 (BLKIOLATENCY_NR_EXP_FACTORS - 1))
175 static const u64 iolatency_exp_factors
[BLKIOLATENCY_NR_EXP_FACTORS
] = {
176 2045, // exp(1/600) - 600 samples
177 2039, // exp(1/240) - 240 samples
178 2031, // exp(1/120) - 120 samples
179 2023, // exp(1/80) - 80 samples
180 2014, // exp(1/60) - 60 samples
183 static inline struct iolatency_grp
*pd_to_lat(struct blkg_policy_data
*pd
)
185 return pd
? container_of(pd
, struct iolatency_grp
, pd
) : NULL
;
188 static inline struct iolatency_grp
*blkg_to_lat(struct blkcg_gq
*blkg
)
190 return pd_to_lat(blkg_to_pd(blkg
, &blkcg_policy_iolatency
));
193 static inline struct blkcg_gq
*lat_to_blkg(struct iolatency_grp
*iolat
)
195 return pd_to_blkg(&iolat
->pd
);
198 static inline void latency_stat_init(struct iolatency_grp
*iolat
,
199 struct latency_stat
*stat
)
205 blk_rq_stat_init(&stat
->rqs
);
208 static inline void latency_stat_sum(struct iolatency_grp
*iolat
,
209 struct latency_stat
*sum
,
210 struct latency_stat
*stat
)
213 sum
->ps
.total
+= stat
->ps
.total
;
214 sum
->ps
.missed
+= stat
->ps
.missed
;
216 blk_rq_stat_sum(&sum
->rqs
, &stat
->rqs
);
219 static inline void latency_stat_record_time(struct iolatency_grp
*iolat
,
222 struct latency_stat
*stat
= get_cpu_ptr(iolat
->stats
);
224 if (req_time
>= iolat
->min_lat_nsec
)
228 blk_rq_stat_add(&stat
->rqs
, req_time
);
232 static inline bool latency_sum_ok(struct iolatency_grp
*iolat
,
233 struct latency_stat
*stat
)
236 u64 thresh
= div64_u64(stat
->ps
.total
, 10);
237 thresh
= max(thresh
, 1ULL);
238 return stat
->ps
.missed
< thresh
;
240 return stat
->rqs
.mean
<= iolat
->min_lat_nsec
;
243 static inline u64
latency_stat_samples(struct iolatency_grp
*iolat
,
244 struct latency_stat
*stat
)
247 return stat
->ps
.total
;
248 return stat
->rqs
.nr_samples
;
251 static inline void iolat_update_total_lat_avg(struct iolatency_grp
*iolat
,
252 struct latency_stat
*stat
)
260 * calc_load() takes in a number stored in fixed point representation.
261 * Because we are using this for IO time in ns, the values stored
262 * are significantly larger than the FIXED_1 denominator (2048).
263 * Therefore, rounding errors in the calculation are negligible and
266 exp_idx
= min_t(int, BLKIOLATENCY_NR_EXP_FACTORS
- 1,
267 div64_u64(iolat
->cur_win_nsec
,
268 BLKIOLATENCY_EXP_BUCKET_SIZE
));
269 iolat
->lat_avg
= calc_load(iolat
->lat_avg
,
270 iolatency_exp_factors
[exp_idx
],
274 static void iolat_cleanup_cb(struct rq_wait
*rqw
, void *private_data
)
276 atomic_dec(&rqw
->inflight
);
280 static bool iolat_acquire_inflight(struct rq_wait
*rqw
, void *private_data
)
282 struct iolatency_grp
*iolat
= private_data
;
283 return rq_wait_inc_below(rqw
, iolat
->rq_depth
.max_depth
);
286 static void __blkcg_iolatency_throttle(struct rq_qos
*rqos
,
287 struct iolatency_grp
*iolat
,
291 struct rq_wait
*rqw
= &iolat
->rq_wait
;
292 unsigned use_delay
= atomic_read(&lat_to_blkg(iolat
)->use_delay
);
295 blkcg_schedule_throttle(rqos
->q
, use_memdelay
);
298 * To avoid priority inversions we want to just take a slot if we are
299 * issuing as root. If we're being killed off there's no point in
300 * delaying things, we may have been killed by OOM so throttling may
301 * make recovery take even longer, so just let the IO's through so the
304 if (issue_as_root
|| fatal_signal_pending(current
)) {
305 atomic_inc(&rqw
->inflight
);
309 rq_qos_wait(rqw
, iolat
, iolat_acquire_inflight
, iolat_cleanup_cb
);
312 #define SCALE_DOWN_FACTOR 2
313 #define SCALE_UP_FACTOR 4
315 static inline unsigned long scale_amount(unsigned long qd
, bool up
)
317 return max(up
? qd
>> SCALE_UP_FACTOR
: qd
>> SCALE_DOWN_FACTOR
, 1UL);
321 * We scale the qd down faster than we scale up, so we need to use this helper
322 * to adjust the scale_cookie accordingly so we don't prematurely get
323 * scale_cookie at DEFAULT_SCALE_COOKIE and unthrottle too much.
325 * Each group has their own local copy of the last scale cookie they saw, so if
326 * the global scale cookie goes up or down they know which way they need to go
327 * based on their last knowledge of it.
329 static void scale_cookie_change(struct blk_iolatency
*blkiolat
,
330 struct child_latency_info
*lat_info
,
333 unsigned long qd
= blkiolat
->rqos
.q
->nr_requests
;
334 unsigned long scale
= scale_amount(qd
, up
);
335 unsigned long old
= atomic_read(&lat_info
->scale_cookie
);
336 unsigned long max_scale
= qd
<< 1;
337 unsigned long diff
= 0;
339 if (old
< DEFAULT_SCALE_COOKIE
)
340 diff
= DEFAULT_SCALE_COOKIE
- old
;
343 if (scale
+ old
> DEFAULT_SCALE_COOKIE
)
344 atomic_set(&lat_info
->scale_cookie
,
345 DEFAULT_SCALE_COOKIE
);
347 atomic_inc(&lat_info
->scale_cookie
);
349 atomic_add(scale
, &lat_info
->scale_cookie
);
352 * We don't want to dig a hole so deep that it takes us hours to
353 * dig out of it. Just enough that we don't throttle/unthrottle
354 * with jagged workloads but can still unthrottle once pressure
355 * has sufficiently dissipated.
358 if (diff
< max_scale
)
359 atomic_dec(&lat_info
->scale_cookie
);
361 atomic_sub(scale
, &lat_info
->scale_cookie
);
367 * Change the queue depth of the iolatency_grp. We add/subtract 1/16th of the
368 * queue depth at a time so we don't get wild swings and hopefully dial in to
369 * fairer distribution of the overall queue depth.
371 static void scale_change(struct iolatency_grp
*iolat
, bool up
)
373 unsigned long qd
= iolat
->blkiolat
->rqos
.q
->nr_requests
;
374 unsigned long scale
= scale_amount(qd
, up
);
375 unsigned long old
= iolat
->rq_depth
.max_depth
;
381 if (old
== 1 && blkcg_unuse_delay(lat_to_blkg(iolat
)))
387 iolat
->rq_depth
.max_depth
= old
;
388 wake_up_all(&iolat
->rq_wait
.wait
);
392 iolat
->rq_depth
.max_depth
= max(old
, 1UL);
396 /* Check our parent and see if the scale cookie has changed. */
397 static void check_scale_change(struct iolatency_grp
*iolat
)
399 struct iolatency_grp
*parent
;
400 struct child_latency_info
*lat_info
;
401 unsigned int cur_cookie
;
402 unsigned int our_cookie
= atomic_read(&iolat
->scale_cookie
);
406 if (lat_to_blkg(iolat
)->parent
== NULL
)
409 parent
= blkg_to_lat(lat_to_blkg(iolat
)->parent
);
413 lat_info
= &parent
->child_lat
;
414 cur_cookie
= atomic_read(&lat_info
->scale_cookie
);
415 scale_lat
= READ_ONCE(lat_info
->scale_lat
);
417 if (cur_cookie
< our_cookie
)
419 else if (cur_cookie
> our_cookie
)
424 if (!atomic_try_cmpxchg(&iolat
->scale_cookie
, &our_cookie
, cur_cookie
)) {
425 /* Somebody beat us to the punch, just bail. */
429 if (direction
< 0 && iolat
->min_lat_nsec
) {
432 if (!scale_lat
|| iolat
->min_lat_nsec
<= scale_lat
)
436 * Sometimes high priority groups are their own worst enemy, so
437 * instead of taking it out on some poor other group that did 5%
438 * or less of the IO's for the last summation just skip this
441 samples_thresh
= lat_info
->nr_samples
* 5;
442 samples_thresh
= max(1ULL, div64_u64(samples_thresh
, 100));
443 if (iolat
->nr_samples
<= samples_thresh
)
447 /* We're as low as we can go. */
448 if (iolat
->rq_depth
.max_depth
== 1 && direction
< 0) {
449 blkcg_use_delay(lat_to_blkg(iolat
));
453 /* We're back to the default cookie, unthrottle all the things. */
454 if (cur_cookie
== DEFAULT_SCALE_COOKIE
) {
455 blkcg_clear_delay(lat_to_blkg(iolat
));
456 iolat
->rq_depth
.max_depth
= UINT_MAX
;
457 wake_up_all(&iolat
->rq_wait
.wait
);
461 scale_change(iolat
, direction
> 0);
464 static void blkcg_iolatency_throttle(struct rq_qos
*rqos
, struct bio
*bio
)
466 struct blk_iolatency
*blkiolat
= BLKIOLATENCY(rqos
);
467 struct blkcg_gq
*blkg
= bio
->bi_blkg
;
468 bool issue_as_root
= bio_issue_as_root_blkg(bio
);
470 if (!blkiolat
->enabled
)
473 while (blkg
&& blkg
->parent
) {
474 struct iolatency_grp
*iolat
= blkg_to_lat(blkg
);
480 check_scale_change(iolat
);
481 __blkcg_iolatency_throttle(rqos
, iolat
, issue_as_root
,
482 (bio
->bi_opf
& REQ_SWAP
) == REQ_SWAP
);
485 if (!timer_pending(&blkiolat
->timer
))
486 mod_timer(&blkiolat
->timer
, jiffies
+ HZ
);
489 static void iolatency_record_time(struct iolatency_grp
*iolat
,
490 struct bio_issue
*issue
, u64 now
,
493 u64 start
= bio_issue_time(issue
);
497 * Have to do this so we are truncated to the correct time that our
498 * issue is truncated to.
500 now
= __bio_issue_time(now
);
505 req_time
= now
- start
;
508 * We don't want to count issue_as_root bio's in the cgroups latency
509 * statistics as it could skew the numbers downwards.
511 if (unlikely(issue_as_root
&& iolat
->rq_depth
.max_depth
!= UINT_MAX
)) {
512 u64 sub
= iolat
->min_lat_nsec
;
514 blkcg_add_delay(lat_to_blkg(iolat
), now
, sub
- req_time
);
518 latency_stat_record_time(iolat
, req_time
);
521 #define BLKIOLATENCY_MIN_ADJUST_TIME (500 * NSEC_PER_MSEC)
522 #define BLKIOLATENCY_MIN_GOOD_SAMPLES 5
524 static void iolatency_check_latencies(struct iolatency_grp
*iolat
, u64 now
)
526 struct blkcg_gq
*blkg
= lat_to_blkg(iolat
);
527 struct iolatency_grp
*parent
;
528 struct child_latency_info
*lat_info
;
529 struct latency_stat stat
;
533 latency_stat_init(iolat
, &stat
);
535 for_each_online_cpu(cpu
) {
536 struct latency_stat
*s
;
537 s
= per_cpu_ptr(iolat
->stats
, cpu
);
538 latency_stat_sum(iolat
, &stat
, s
);
539 latency_stat_init(iolat
, s
);
543 parent
= blkg_to_lat(blkg
->parent
);
547 lat_info
= &parent
->child_lat
;
549 iolat_update_total_lat_avg(iolat
, &stat
);
551 /* Everything is ok and we don't need to adjust the scale. */
552 if (latency_sum_ok(iolat
, &stat
) &&
553 atomic_read(&lat_info
->scale_cookie
) == DEFAULT_SCALE_COOKIE
)
556 /* Somebody beat us to the punch, just bail. */
557 spin_lock_irqsave(&lat_info
->lock
, flags
);
559 latency_stat_sum(iolat
, &iolat
->cur_stat
, &stat
);
560 lat_info
->nr_samples
-= iolat
->nr_samples
;
561 lat_info
->nr_samples
+= latency_stat_samples(iolat
, &iolat
->cur_stat
);
562 iolat
->nr_samples
= latency_stat_samples(iolat
, &iolat
->cur_stat
);
564 if ((lat_info
->last_scale_event
>= now
||
565 now
- lat_info
->last_scale_event
< BLKIOLATENCY_MIN_ADJUST_TIME
))
568 if (latency_sum_ok(iolat
, &iolat
->cur_stat
) &&
569 latency_sum_ok(iolat
, &stat
)) {
570 if (latency_stat_samples(iolat
, &iolat
->cur_stat
) <
571 BLKIOLATENCY_MIN_GOOD_SAMPLES
)
573 if (lat_info
->scale_grp
== iolat
) {
574 lat_info
->last_scale_event
= now
;
575 scale_cookie_change(iolat
->blkiolat
, lat_info
, true);
577 } else if (lat_info
->scale_lat
== 0 ||
578 lat_info
->scale_lat
>= iolat
->min_lat_nsec
) {
579 lat_info
->last_scale_event
= now
;
580 if (!lat_info
->scale_grp
||
581 lat_info
->scale_lat
> iolat
->min_lat_nsec
) {
582 WRITE_ONCE(lat_info
->scale_lat
, iolat
->min_lat_nsec
);
583 lat_info
->scale_grp
= iolat
;
585 scale_cookie_change(iolat
->blkiolat
, lat_info
, false);
587 latency_stat_init(iolat
, &iolat
->cur_stat
);
589 spin_unlock_irqrestore(&lat_info
->lock
, flags
);
592 static void blkcg_iolatency_done_bio(struct rq_qos
*rqos
, struct bio
*bio
)
594 struct blkcg_gq
*blkg
;
596 struct iolatency_grp
*iolat
;
599 bool issue_as_root
= bio_issue_as_root_blkg(bio
);
603 if (!blkg
|| !bio_flagged(bio
, BIO_QOS_THROTTLED
))
606 iolat
= blkg_to_lat(bio
->bi_blkg
);
610 if (!iolat
->blkiolat
->enabled
)
613 now
= ktime_to_ns(ktime_get());
614 while (blkg
&& blkg
->parent
) {
615 iolat
= blkg_to_lat(blkg
);
620 rqw
= &iolat
->rq_wait
;
622 inflight
= atomic_dec_return(&rqw
->inflight
);
623 WARN_ON_ONCE(inflight
< 0);
625 * If bi_status is BLK_STS_AGAIN, the bio wasn't actually
626 * submitted, so do not account for it.
628 if (iolat
->min_lat_nsec
&& bio
->bi_status
!= BLK_STS_AGAIN
) {
629 iolatency_record_time(iolat
, &bio
->bi_issue
, now
,
631 window_start
= atomic64_read(&iolat
->window_start
);
632 if (now
> window_start
&&
633 (now
- window_start
) >= iolat
->cur_win_nsec
) {
634 if (atomic64_try_cmpxchg(&iolat
->window_start
,
636 iolatency_check_latencies(iolat
, now
);
644 static void blkcg_iolatency_exit(struct rq_qos
*rqos
)
646 struct blk_iolatency
*blkiolat
= BLKIOLATENCY(rqos
);
648 del_timer_sync(&blkiolat
->timer
);
649 flush_work(&blkiolat
->enable_work
);
650 blkcg_deactivate_policy(rqos
->q
, &blkcg_policy_iolatency
);
654 static struct rq_qos_ops blkcg_iolatency_ops
= {
655 .throttle
= blkcg_iolatency_throttle
,
656 .done_bio
= blkcg_iolatency_done_bio
,
657 .exit
= blkcg_iolatency_exit
,
660 static void blkiolatency_timer_fn(struct timer_list
*t
)
662 struct blk_iolatency
*blkiolat
= from_timer(blkiolat
, t
, timer
);
663 struct blkcg_gq
*blkg
;
664 struct cgroup_subsys_state
*pos_css
;
665 u64 now
= ktime_to_ns(ktime_get());
668 blkg_for_each_descendant_pre(blkg
, pos_css
,
669 blkiolat
->rqos
.q
->root_blkg
) {
670 struct iolatency_grp
*iolat
;
671 struct child_latency_info
*lat_info
;
676 * We could be exiting, don't access the pd unless we have a
679 if (!blkg_tryget(blkg
))
682 iolat
= blkg_to_lat(blkg
);
686 lat_info
= &iolat
->child_lat
;
687 cookie
= atomic_read(&lat_info
->scale_cookie
);
689 if (cookie
>= DEFAULT_SCALE_COOKIE
)
692 spin_lock_irqsave(&lat_info
->lock
, flags
);
693 if (lat_info
->last_scale_event
>= now
)
697 * We scaled down but don't have a scale_grp, scale up and carry
700 if (lat_info
->scale_grp
== NULL
) {
701 scale_cookie_change(iolat
->blkiolat
, lat_info
, true);
706 * It's been 5 seconds since our last scale event, clear the
707 * scale grp in case the group that needed the scale down isn't
708 * doing any IO currently.
710 if (now
- lat_info
->last_scale_event
>=
711 ((u64
)NSEC_PER_SEC
* 5))
712 lat_info
->scale_grp
= NULL
;
714 spin_unlock_irqrestore(&lat_info
->lock
, flags
);
722 * blkiolatency_enable_work_fn - Enable or disable iolatency on the device
723 * @work: enable_work of the blk_iolatency of interest
725 * iolatency needs to keep track of the number of in-flight IOs per cgroup. This
726 * is relatively expensive as it involves walking up the hierarchy twice for
727 * every IO. Thus, if iolatency is not enabled in any cgroup for the device, we
728 * want to disable the in-flight tracking.
730 * We have to make sure that the counting is balanced - we don't want to leak
731 * the in-flight counts by disabling accounting in the completion path while IOs
732 * are in flight. This is achieved by ensuring that no IO is in flight by
733 * freezing the queue while flipping ->enabled. As this requires a sleepable
734 * context, ->enabled flipping is punted to this work function.
736 static void blkiolatency_enable_work_fn(struct work_struct
*work
)
738 struct blk_iolatency
*blkiolat
= container_of(work
, struct blk_iolatency
,
743 * There can only be one instance of this function running for @blkiolat
744 * and it's guaranteed to be executed at least once after the latest
745 * ->enabled_cnt modification. Acting on the latest ->enable_cnt is
748 * Also, we know @blkiolat is safe to access as ->enable_work is flushed
749 * in blkcg_iolatency_exit().
751 enabled
= atomic_read(&blkiolat
->enable_cnt
);
752 if (enabled
!= blkiolat
->enabled
) {
753 blk_mq_freeze_queue(blkiolat
->rqos
.q
);
754 blkiolat
->enabled
= enabled
;
755 blk_mq_unfreeze_queue(blkiolat
->rqos
.q
);
759 int blk_iolatency_init(struct request_queue
*q
)
761 struct blk_iolatency
*blkiolat
;
765 blkiolat
= kzalloc(sizeof(*blkiolat
), GFP_KERNEL
);
769 rqos
= &blkiolat
->rqos
;
770 rqos
->id
= RQ_QOS_LATENCY
;
771 rqos
->ops
= &blkcg_iolatency_ops
;
774 ret
= rq_qos_add(q
, rqos
);
777 ret
= blkcg_activate_policy(q
, &blkcg_policy_iolatency
);
781 timer_setup(&blkiolat
->timer
, blkiolatency_timer_fn
, 0);
782 INIT_WORK(&blkiolat
->enable_work
, blkiolatency_enable_work_fn
);
793 static void iolatency_set_min_lat_nsec(struct blkcg_gq
*blkg
, u64 val
)
795 struct iolatency_grp
*iolat
= blkg_to_lat(blkg
);
796 struct blk_iolatency
*blkiolat
= iolat
->blkiolat
;
797 u64 oldval
= iolat
->min_lat_nsec
;
799 iolat
->min_lat_nsec
= val
;
800 iolat
->cur_win_nsec
= max_t(u64
, val
<< 4, BLKIOLATENCY_MIN_WIN_SIZE
);
801 iolat
->cur_win_nsec
= min_t(u64
, iolat
->cur_win_nsec
,
802 BLKIOLATENCY_MAX_WIN_SIZE
);
804 if (!oldval
&& val
) {
805 if (atomic_inc_return(&blkiolat
->enable_cnt
) == 1)
806 schedule_work(&blkiolat
->enable_work
);
808 if (oldval
&& !val
) {
809 blkcg_clear_delay(blkg
);
810 if (atomic_dec_return(&blkiolat
->enable_cnt
) == 0)
811 schedule_work(&blkiolat
->enable_work
);
815 static void iolatency_clear_scaling(struct blkcg_gq
*blkg
)
818 struct iolatency_grp
*iolat
= blkg_to_lat(blkg
->parent
);
819 struct child_latency_info
*lat_info
;
823 lat_info
= &iolat
->child_lat
;
824 spin_lock(&lat_info
->lock
);
825 atomic_set(&lat_info
->scale_cookie
, DEFAULT_SCALE_COOKIE
);
826 lat_info
->last_scale_event
= 0;
827 lat_info
->scale_grp
= NULL
;
828 lat_info
->scale_lat
= 0;
829 spin_unlock(&lat_info
->lock
);
833 static ssize_t
iolatency_set_limit(struct kernfs_open_file
*of
, char *buf
,
834 size_t nbytes
, loff_t off
)
836 struct blkcg
*blkcg
= css_to_blkcg(of_css(of
));
837 struct blkcg_gq
*blkg
;
838 struct blkg_conf_ctx ctx
;
839 struct iolatency_grp
*iolat
;
845 ret
= blkg_conf_prep(blkcg
, &blkcg_policy_iolatency
, buf
, &ctx
);
849 iolat
= blkg_to_lat(ctx
.blkg
);
853 while ((tok
= strsep(&p
, " "))) {
855 char val
[21]; /* 18446744073709551616 */
857 if (sscanf(tok
, "%15[^=]=%20s", key
, val
) != 2)
860 if (!strcmp(key
, "target")) {
863 if (!strcmp(val
, "max"))
865 else if (sscanf(val
, "%llu", &v
) == 1)
866 lat_val
= v
* NSEC_PER_USEC
;
874 /* Walk up the tree to see if our new val is lower than it should be. */
876 oldval
= iolat
->min_lat_nsec
;
878 iolatency_set_min_lat_nsec(blkg
, lat_val
);
879 if (oldval
!= iolat
->min_lat_nsec
)
880 iolatency_clear_scaling(blkg
);
883 blkg_conf_finish(&ctx
);
884 return ret
?: nbytes
;
887 static u64
iolatency_prfill_limit(struct seq_file
*sf
,
888 struct blkg_policy_data
*pd
, int off
)
890 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
891 const char *dname
= blkg_dev_name(pd
->blkg
);
893 if (!dname
|| !iolat
->min_lat_nsec
)
895 seq_printf(sf
, "%s target=%llu\n",
896 dname
, div_u64(iolat
->min_lat_nsec
, NSEC_PER_USEC
));
900 static int iolatency_print_limit(struct seq_file
*sf
, void *v
)
902 blkcg_print_blkgs(sf
, css_to_blkcg(seq_css(sf
)),
903 iolatency_prfill_limit
,
904 &blkcg_policy_iolatency
, seq_cft(sf
)->private, false);
908 static void iolatency_ssd_stat(struct iolatency_grp
*iolat
, struct seq_file
*s
)
910 struct latency_stat stat
;
913 latency_stat_init(iolat
, &stat
);
915 for_each_online_cpu(cpu
) {
916 struct latency_stat
*s
;
917 s
= per_cpu_ptr(iolat
->stats
, cpu
);
918 latency_stat_sum(iolat
, &stat
, s
);
922 if (iolat
->rq_depth
.max_depth
== UINT_MAX
)
923 seq_printf(s
, " missed=%llu total=%llu depth=max",
924 (unsigned long long)stat
.ps
.missed
,
925 (unsigned long long)stat
.ps
.total
);
927 seq_printf(s
, " missed=%llu total=%llu depth=%u",
928 (unsigned long long)stat
.ps
.missed
,
929 (unsigned long long)stat
.ps
.total
,
930 iolat
->rq_depth
.max_depth
);
933 static void iolatency_pd_stat(struct blkg_policy_data
*pd
, struct seq_file
*s
)
935 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
936 unsigned long long avg_lat
;
937 unsigned long long cur_win
;
939 if (!blkcg_debug_stats
)
943 return iolatency_ssd_stat(iolat
, s
);
945 avg_lat
= div64_u64(iolat
->lat_avg
, NSEC_PER_USEC
);
946 cur_win
= div64_u64(iolat
->cur_win_nsec
, NSEC_PER_MSEC
);
947 if (iolat
->rq_depth
.max_depth
== UINT_MAX
)
948 seq_printf(s
, " depth=max avg_lat=%llu win=%llu",
951 seq_printf(s
, " depth=%u avg_lat=%llu win=%llu",
952 iolat
->rq_depth
.max_depth
, avg_lat
, cur_win
);
955 static struct blkg_policy_data
*iolatency_pd_alloc(gfp_t gfp
,
956 struct request_queue
*q
,
959 struct iolatency_grp
*iolat
;
961 iolat
= kzalloc_node(sizeof(*iolat
), gfp
, q
->node
);
964 iolat
->stats
= __alloc_percpu_gfp(sizeof(struct latency_stat
),
965 __alignof__(struct latency_stat
), gfp
);
973 static void iolatency_pd_init(struct blkg_policy_data
*pd
)
975 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
976 struct blkcg_gq
*blkg
= lat_to_blkg(iolat
);
977 struct rq_qos
*rqos
= blkcg_rq_qos(blkg
->q
);
978 struct blk_iolatency
*blkiolat
= BLKIOLATENCY(rqos
);
979 u64 now
= ktime_to_ns(ktime_get());
982 if (blk_queue_nonrot(blkg
->q
))
987 for_each_possible_cpu(cpu
) {
988 struct latency_stat
*stat
;
989 stat
= per_cpu_ptr(iolat
->stats
, cpu
);
990 latency_stat_init(iolat
, stat
);
993 latency_stat_init(iolat
, &iolat
->cur_stat
);
994 rq_wait_init(&iolat
->rq_wait
);
995 spin_lock_init(&iolat
->child_lat
.lock
);
996 iolat
->rq_depth
.queue_depth
= blkg
->q
->nr_requests
;
997 iolat
->rq_depth
.max_depth
= UINT_MAX
;
998 iolat
->rq_depth
.default_depth
= iolat
->rq_depth
.queue_depth
;
999 iolat
->blkiolat
= blkiolat
;
1000 iolat
->cur_win_nsec
= 100 * NSEC_PER_MSEC
;
1001 atomic64_set(&iolat
->window_start
, now
);
1004 * We init things in list order, so the pd for the parent may not be
1005 * init'ed yet for whatever reason.
1007 if (blkg
->parent
&& blkg_to_pd(blkg
->parent
, &blkcg_policy_iolatency
)) {
1008 struct iolatency_grp
*parent
= blkg_to_lat(blkg
->parent
);
1009 atomic_set(&iolat
->scale_cookie
,
1010 atomic_read(&parent
->child_lat
.scale_cookie
));
1012 atomic_set(&iolat
->scale_cookie
, DEFAULT_SCALE_COOKIE
);
1015 atomic_set(&iolat
->child_lat
.scale_cookie
, DEFAULT_SCALE_COOKIE
);
1018 static void iolatency_pd_offline(struct blkg_policy_data
*pd
)
1020 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
1021 struct blkcg_gq
*blkg
= lat_to_blkg(iolat
);
1023 iolatency_set_min_lat_nsec(blkg
, 0);
1024 iolatency_clear_scaling(blkg
);
1027 static void iolatency_pd_free(struct blkg_policy_data
*pd
)
1029 struct iolatency_grp
*iolat
= pd_to_lat(pd
);
1030 free_percpu(iolat
->stats
);
1034 static struct cftype iolatency_files
[] = {
1037 .flags
= CFTYPE_NOT_ON_ROOT
,
1038 .seq_show
= iolatency_print_limit
,
1039 .write
= iolatency_set_limit
,
1044 static struct blkcg_policy blkcg_policy_iolatency
= {
1045 .dfl_cftypes
= iolatency_files
,
1046 .pd_alloc_fn
= iolatency_pd_alloc
,
1047 .pd_init_fn
= iolatency_pd_init
,
1048 .pd_offline_fn
= iolatency_pd_offline
,
1049 .pd_free_fn
= iolatency_pd_free
,
1050 .pd_stat_fn
= iolatency_pd_stat
,
1053 static int __init
iolatency_init(void)
1055 return blkcg_policy_register(&blkcg_policy_iolatency
);
1058 static void __exit
iolatency_exit(void)
1060 blkcg_policy_unregister(&blkcg_policy_iolatency
);
1063 module_init(iolatency_init
);
1064 module_exit(iolatency_exit
);