]>
Commit | Line | Data |
---|---|---|
cf43e6be JA |
1 | /* |
2 | * Block stat tracking code | |
3 | * | |
4 | * Copyright (C) 2016 Jens Axboe | |
5 | */ | |
6 | #include <linux/kernel.h> | |
34dbad5d | 7 | #include <linux/rculist.h> |
cf43e6be JA |
8 | #include <linux/blk-mq.h> |
9 | ||
10 | #include "blk-stat.h" | |
11 | #include "blk-mq.h" | |
b9147dd1 | 12 | #include "blk.h" |
cf43e6be | 13 | |
34dbad5d OS |
14 | struct blk_queue_stats { |
15 | struct list_head callbacks; | |
16 | spinlock_t lock; | |
b9147dd1 | 17 | bool enable_accounting; |
34dbad5d OS |
18 | }; |
19 | ||
34dbad5d OS |
20 | static void blk_stat_init(struct blk_rq_stat *stat) |
21 | { | |
22 | stat->min = -1ULL; | |
23 | stat->max = stat->nr_samples = stat->mean = 0; | |
eca8b53a | 24 | stat->batch = 0; |
cf43e6be JA |
25 | } |
26 | ||
eca8b53a | 27 | /* src is a per-cpu stat, mean isn't initialized */ |
cf43e6be JA |
28 | static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) |
29 | { | |
30 | if (!src->nr_samples) | |
31 | return; | |
32 | ||
cf43e6be JA |
33 | dst->min = min(dst->min, src->min); |
34 | dst->max = max(dst->max, src->max); | |
35 | ||
eca8b53a SL |
36 | dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples, |
37 | dst->nr_samples + src->nr_samples); | |
38 | ||
cf43e6be JA |
39 | dst->nr_samples += src->nr_samples; |
40 | } | |
41 | ||
34dbad5d | 42 | static void __blk_stat_add(struct blk_rq_stat *stat, u64 value) |
cf43e6be | 43 | { |
34dbad5d OS |
44 | stat->min = min(stat->min, value); |
45 | stat->max = max(stat->max, value); | |
34dbad5d | 46 | stat->batch += value; |
eca8b53a | 47 | stat->nr_samples++; |
cf43e6be JA |
48 | } |
49 | ||
34dbad5d | 50 | void blk_stat_add(struct request *rq) |
cf43e6be | 51 | { |
34dbad5d OS |
52 | struct request_queue *q = rq->q; |
53 | struct blk_stat_callback *cb; | |
54 | struct blk_rq_stat *stat; | |
55 | int bucket; | |
eca8b53a | 56 | u64 now, value; |
34dbad5d | 57 | |
544ccc8d OS |
58 | now = ktime_get_ns(); |
59 | value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0; | |
34dbad5d | 60 | |
b9147dd1 SL |
61 | blk_throtl_stat_add(rq, value); |
62 | ||
34dbad5d OS |
63 | rcu_read_lock(); |
64 | list_for_each_entry_rcu(cb, &q->stats->callbacks, list) { | |
d3738123 JA |
65 | if (!blk_stat_is_active(cb)) |
66 | continue; | |
67 | ||
68 | bucket = cb->bucket_fn(rq); | |
69 | if (bucket < 0) | |
70 | continue; | |
71 | ||
72 | stat = &get_cpu_ptr(cb->cpu_stat)[bucket]; | |
73 | __blk_stat_add(stat, value); | |
74 | put_cpu_ptr(cb->cpu_stat); | |
cf43e6be | 75 | } |
34dbad5d | 76 | rcu_read_unlock(); |
cf43e6be JA |
77 | } |
78 | ||
e99e88a9 | 79 | static void blk_stat_timer_fn(struct timer_list *t) |
cf43e6be | 80 | { |
e99e88a9 | 81 | struct blk_stat_callback *cb = from_timer(cb, t, timer); |
34dbad5d OS |
82 | unsigned int bucket; |
83 | int cpu; | |
cf43e6be | 84 | |
34dbad5d OS |
85 | for (bucket = 0; bucket < cb->buckets; bucket++) |
86 | blk_stat_init(&cb->stat[bucket]); | |
cf43e6be | 87 | |
34dbad5d OS |
88 | for_each_online_cpu(cpu) { |
89 | struct blk_rq_stat *cpu_stat; | |
7cd54aa8 | 90 | |
34dbad5d OS |
91 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
92 | for (bucket = 0; bucket < cb->buckets; bucket++) { | |
93 | blk_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]); | |
94 | blk_stat_init(&cpu_stat[bucket]); | |
cf43e6be | 95 | } |
34dbad5d | 96 | } |
cf43e6be | 97 | |
34dbad5d | 98 | cb->timer_fn(cb); |
cf43e6be JA |
99 | } |
100 | ||
34dbad5d OS |
101 | struct blk_stat_callback * |
102 | blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), | |
a37244e4 | 103 | int (*bucket_fn)(const struct request *), |
34dbad5d | 104 | unsigned int buckets, void *data) |
cf43e6be | 105 | { |
34dbad5d | 106 | struct blk_stat_callback *cb; |
cf43e6be | 107 | |
34dbad5d OS |
108 | cb = kmalloc(sizeof(*cb), GFP_KERNEL); |
109 | if (!cb) | |
110 | return NULL; | |
cf43e6be | 111 | |
34dbad5d OS |
112 | cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat), |
113 | GFP_KERNEL); | |
114 | if (!cb->stat) { | |
115 | kfree(cb); | |
116 | return NULL; | |
117 | } | |
118 | cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat), | |
119 | __alignof__(struct blk_rq_stat)); | |
120 | if (!cb->cpu_stat) { | |
121 | kfree(cb->stat); | |
122 | kfree(cb); | |
123 | return NULL; | |
124 | } | |
cf43e6be | 125 | |
34dbad5d OS |
126 | cb->timer_fn = timer_fn; |
127 | cb->bucket_fn = bucket_fn; | |
128 | cb->data = data; | |
129 | cb->buckets = buckets; | |
e99e88a9 | 130 | timer_setup(&cb->timer, blk_stat_timer_fn, 0); |
34dbad5d OS |
131 | |
132 | return cb; | |
cf43e6be | 133 | } |
34dbad5d | 134 | EXPORT_SYMBOL_GPL(blk_stat_alloc_callback); |
cf43e6be | 135 | |
34dbad5d OS |
136 | void blk_stat_add_callback(struct request_queue *q, |
137 | struct blk_stat_callback *cb) | |
cf43e6be | 138 | { |
34dbad5d OS |
139 | unsigned int bucket; |
140 | int cpu; | |
cf43e6be | 141 | |
34dbad5d OS |
142 | for_each_possible_cpu(cpu) { |
143 | struct blk_rq_stat *cpu_stat; | |
cf43e6be | 144 | |
34dbad5d OS |
145 | cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
146 | for (bucket = 0; bucket < cb->buckets; bucket++) | |
147 | blk_stat_init(&cpu_stat[bucket]); | |
148 | } | |
cf43e6be | 149 | |
34dbad5d OS |
150 | spin_lock(&q->stats->lock); |
151 | list_add_tail_rcu(&cb->list, &q->stats->callbacks); | |
7dfdbc73 | 152 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
34dbad5d OS |
153 | spin_unlock(&q->stats->lock); |
154 | } | |
155 | EXPORT_SYMBOL_GPL(blk_stat_add_callback); | |
cf43e6be | 156 | |
34dbad5d OS |
157 | void blk_stat_remove_callback(struct request_queue *q, |
158 | struct blk_stat_callback *cb) | |
159 | { | |
160 | spin_lock(&q->stats->lock); | |
161 | list_del_rcu(&cb->list); | |
b9147dd1 | 162 | if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting) |
7dfdbc73 | 163 | blk_queue_flag_clear(QUEUE_FLAG_STATS, q); |
34dbad5d | 164 | spin_unlock(&q->stats->lock); |
cf43e6be | 165 | |
34dbad5d | 166 | del_timer_sync(&cb->timer); |
cf43e6be | 167 | } |
34dbad5d | 168 | EXPORT_SYMBOL_GPL(blk_stat_remove_callback); |
cf43e6be | 169 | |
34dbad5d | 170 | static void blk_stat_free_callback_rcu(struct rcu_head *head) |
cf43e6be | 171 | { |
34dbad5d OS |
172 | struct blk_stat_callback *cb; |
173 | ||
174 | cb = container_of(head, struct blk_stat_callback, rcu); | |
175 | free_percpu(cb->cpu_stat); | |
176 | kfree(cb->stat); | |
177 | kfree(cb); | |
cf43e6be JA |
178 | } |
179 | ||
34dbad5d | 180 | void blk_stat_free_callback(struct blk_stat_callback *cb) |
cf43e6be | 181 | { |
a83b576c JA |
182 | if (cb) |
183 | call_rcu(&cb->rcu, blk_stat_free_callback_rcu); | |
cf43e6be | 184 | } |
34dbad5d | 185 | EXPORT_SYMBOL_GPL(blk_stat_free_callback); |
cf43e6be | 186 | |
b9147dd1 SL |
187 | void blk_stat_enable_accounting(struct request_queue *q) |
188 | { | |
189 | spin_lock(&q->stats->lock); | |
190 | q->stats->enable_accounting = true; | |
7dfdbc73 | 191 | blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
b9147dd1 SL |
192 | spin_unlock(&q->stats->lock); |
193 | } | |
194 | ||
34dbad5d | 195 | struct blk_queue_stats *blk_alloc_queue_stats(void) |
cf43e6be | 196 | { |
34dbad5d OS |
197 | struct blk_queue_stats *stats; |
198 | ||
199 | stats = kmalloc(sizeof(*stats), GFP_KERNEL); | |
200 | if (!stats) | |
201 | return NULL; | |
202 | ||
203 | INIT_LIST_HEAD(&stats->callbacks); | |
204 | spin_lock_init(&stats->lock); | |
b9147dd1 | 205 | stats->enable_accounting = false; |
34dbad5d OS |
206 | |
207 | return stats; | |
208 | } | |
209 | ||
210 | void blk_free_queue_stats(struct blk_queue_stats *stats) | |
211 | { | |
212 | if (!stats) | |
213 | return; | |
214 | ||
215 | WARN_ON(!list_empty(&stats->callbacks)); | |
cf43e6be | 216 | |
34dbad5d | 217 | kfree(stats); |
cf43e6be | 218 | } |