]> git.ipfire.org Git - thirdparty/kernel/linux.git/blame - tools/perf/util/ordered-events.c
Merge tag 'vfio-v4.20-rc1.v2' of git://github.com/awilliam/linux-vfio
[thirdparty/kernel/linux.git] / tools / perf / util / ordered-events.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
a43783ae 2#include <errno.h>
fd20e811 3#include <inttypes.h>
5f86b80b 4#include <linux/list.h>
cee3ab9c 5#include <linux/compiler.h>
54bf53b1 6#include <linux/string.h>
5f86b80b 7#include "ordered-events.h"
5f86b80b
JO
8#include "session.h"
9#include "asm/bug.h"
10#include "debug.h"
11
cee3ab9c
JO
12#define pr_N(n, fmt, ...) \
13 eprintf(n, debug_ordered_events, fmt, ##__VA_ARGS__)
14
15#define pr(fmt, ...) pr_N(1, pr_fmt(fmt), ##__VA_ARGS__)
16
5f86b80b
JO
17static void queue_event(struct ordered_events *oe, struct ordered_event *new)
18{
19 struct ordered_event *last = oe->last;
20 u64 timestamp = new->timestamp;
21 struct list_head *p;
22
23 ++oe->nr_events;
24 oe->last = new;
25
cee3ab9c
JO
26 pr_oe_time2(timestamp, "queue_event nr_events %u\n", oe->nr_events);
27
5f86b80b
JO
28 if (!last) {
29 list_add(&new->list, &oe->events);
30 oe->max_timestamp = timestamp;
31 return;
32 }
33
34 /*
35 * last event might point to some random place in the list as it's
36 * the last queued event. We expect that the new event is close to
37 * this.
38 */
39 if (last->timestamp <= timestamp) {
40 while (last->timestamp <= timestamp) {
41 p = last->list.next;
42 if (p == &oe->events) {
43 list_add_tail(&new->list, &oe->events);
44 oe->max_timestamp = timestamp;
45 return;
46 }
47 last = list_entry(p, struct ordered_event, list);
48 }
49 list_add_tail(&new->list, &last->list);
50 } else {
51 while (last->timestamp > timestamp) {
52 p = last->list.prev;
53 if (p == &oe->events) {
54 list_add(&new->list, &oe->events);
55 return;
56 }
57 last = list_entry(p, struct ordered_event, list);
58 }
59 list_add(&new->list, &last->list);
60 }
61}
62
54bf53b1
AY
63static union perf_event *__dup_event(struct ordered_events *oe,
64 union perf_event *event)
65{
66 union perf_event *new_event = NULL;
67
68 if (oe->cur_alloc_size < oe->max_alloc_size) {
69 new_event = memdup(event, event->header.size);
70 if (new_event)
71 oe->cur_alloc_size += event->header.size;
72 }
73
74 return new_event;
75}
76
77static union perf_event *dup_event(struct ordered_events *oe,
78 union perf_event *event)
79{
80 return oe->copy_on_queue ? __dup_event(oe, event) : event;
81}
82
d5ceb62b 83static void __free_dup_event(struct ordered_events *oe, union perf_event *event)
54bf53b1 84{
d5ceb62b 85 if (event) {
54bf53b1
AY
86 oe->cur_alloc_size -= event->header.size;
87 free(event);
88 }
89}
90
d5ceb62b
JO
91static void free_dup_event(struct ordered_events *oe, union perf_event *event)
92{
93 if (oe->copy_on_queue)
94 __free_dup_event(oe, event);
95}
96
5f86b80b 97#define MAX_SAMPLE_BUFFER (64 * 1024 / sizeof(struct ordered_event))
54bf53b1
AY
98static struct ordered_event *alloc_event(struct ordered_events *oe,
99 union perf_event *event)
5f86b80b
JO
100{
101 struct list_head *cache = &oe->cache;
102 struct ordered_event *new = NULL;
54bf53b1 103 union perf_event *new_event;
53da12e0 104 size_t size;
54bf53b1
AY
105
106 new_event = dup_event(oe, event);
107 if (!new_event)
108 return NULL;
5f86b80b 109
d5ceb62b
JO
110 /*
111 * We maintain the following scheme of buffers for ordered
112 * event allocation:
113 *
114 * to_free list -> buffer1 (64K)
115 * buffer2 (64K)
116 * ...
117 *
118 * Each buffer keeps an array of ordered events objects:
119 * buffer -> event[0]
120 * event[1]
121 * ...
122 *
123 * Each allocated ordered event is linked to one of
124 * following lists:
125 * - time ordered list 'events'
126 * - list of currently removed events 'cache'
127 *
128 * Allocation of the ordered event uses the following order
129 * to get the memory:
130 * - use recently removed object from 'cache' list
131 * - use available object in current allocation buffer
132 * - allocate new buffer if the current buffer is full
133 *
134 * Removal of ordered event object moves it from events to
135 * the cache list.
136 */
53da12e0
JO
137 size = sizeof(*oe->buffer) + MAX_SAMPLE_BUFFER * sizeof(*new);
138
5f86b80b
JO
139 if (!list_empty(cache)) {
140 new = list_entry(cache->next, struct ordered_event, list);
141 list_del(&new->list);
142 } else if (oe->buffer) {
d5ceb62b 143 new = &oe->buffer->event[oe->buffer_idx];
5f86b80b
JO
144 if (++oe->buffer_idx == MAX_SAMPLE_BUFFER)
145 oe->buffer = NULL;
53da12e0 146 } else if ((oe->cur_alloc_size + size) < oe->max_alloc_size) {
5f86b80b 147 oe->buffer = malloc(size);
54bf53b1
AY
148 if (!oe->buffer) {
149 free_dup_event(oe, new_event);
5f86b80b 150 return NULL;
54bf53b1 151 }
5f86b80b 152
cee3ab9c
JO
153 pr("alloc size %" PRIu64 "B (+%zu), max %" PRIu64 "B\n",
154 oe->cur_alloc_size, size, oe->max_alloc_size);
155
5f86b80b
JO
156 oe->cur_alloc_size += size;
157 list_add(&oe->buffer->list, &oe->to_free);
158
d5ceb62b
JO
159 oe->buffer_idx = 1;
160 new = &oe->buffer->event[0];
cee3ab9c
JO
161 } else {
162 pr("allocation limit reached %" PRIu64 "B\n", oe->max_alloc_size);
d5ceb62b 163 return NULL;
5f86b80b
JO
164 }
165
54bf53b1 166 new->event = new_event;
5f86b80b
JO
167 return new;
168}
169
4a6b362f
ACM
170static struct ordered_event *
171ordered_events__new_event(struct ordered_events *oe, u64 timestamp,
54bf53b1 172 union perf_event *event)
5f86b80b
JO
173{
174 struct ordered_event *new;
175
54bf53b1 176 new = alloc_event(oe, event);
5f86b80b
JO
177 if (new) {
178 new->timestamp = timestamp;
179 queue_event(oe, new);
180 }
181
182 return new;
183}
184
185void ordered_events__delete(struct ordered_events *oe, struct ordered_event *event)
186{
fa4e5c67 187 list_move(&event->list, &oe->cache);
5f86b80b 188 oe->nr_events--;
54bf53b1 189 free_dup_event(oe, event->event);
1e0d4f02 190 event->event = NULL;
5f86b80b
JO
191}
192
4a6b362f 193int ordered_events__queue(struct ordered_events *oe, union perf_event *event,
dc83e139 194 u64 timestamp, u64 file_offset)
4a6b362f 195{
4a6b362f
ACM
196 struct ordered_event *oevent;
197
198 if (!timestamp || timestamp == ~0ULL)
199 return -ETIME;
200
201 if (timestamp < oe->last_flush) {
202 pr_oe_time(timestamp, "out of order event\n");
203 pr_oe_time(oe->last_flush, "last flush, last_flush_type %d\n",
204 oe->last_flush_type);
205
9870d780 206 oe->nr_unordered_events++;
4a6b362f
ACM
207 }
208
209 oevent = ordered_events__new_event(oe, timestamp, event);
210 if (!oevent) {
211 ordered_events__flush(oe, OE_FLUSH__HALF);
212 oevent = ordered_events__new_event(oe, timestamp, event);
213 }
214
215 if (!oevent)
216 return -ENOMEM;
217
218 oevent->file_offset = file_offset;
219 return 0;
220}
221
b7b61cbe 222static int __ordered_events__flush(struct ordered_events *oe)
5f86b80b 223{
5f86b80b
JO
224 struct list_head *head = &oe->events;
225 struct ordered_event *tmp, *iter;
5f86b80b
JO
226 u64 limit = oe->next_flush;
227 u64 last_ts = oe->last ? oe->last->timestamp : 0ULL;
228 bool show_progress = limit == ULLONG_MAX;
229 struct ui_progress prog;
230 int ret;
231
28083681 232 if (!limit)
5f86b80b
JO
233 return 0;
234
235 if (show_progress)
236 ui_progress__init(&prog, oe->nr_events, "Processing time ordered events...");
237
238 list_for_each_entry_safe(iter, tmp, head, list) {
239 if (session_done())
240 return 0;
241
242 if (iter->timestamp > limit)
243 break;
9870d780 244 ret = oe->deliver(oe, iter);
5f86b80b 245 if (ret)
9870d780 246 return ret;
5f86b80b
JO
247
248 ordered_events__delete(oe, iter);
249 oe->last_flush = iter->timestamp;
250
251 if (show_progress)
252 ui_progress__update(&prog, 1);
253 }
254
255 if (list_empty(head))
256 oe->last = NULL;
257 else if (last_ts <= limit)
258 oe->last = list_entry(head->prev, struct ordered_event, list);
259
5c9ce1e6
ACM
260 if (show_progress)
261 ui_progress__finish();
262
5f86b80b
JO
263 return 0;
264}
265
b7b61cbe 266int ordered_events__flush(struct ordered_events *oe, enum oe_flush how)
5f86b80b 267{
cee3ab9c 268 static const char * const str[] = {
b0a45203 269 "NONE",
cee3ab9c
JO
270 "FINAL",
271 "ROUND",
272 "HALF ",
273 };
5f86b80b
JO
274 int err;
275
28083681
ACM
276 if (oe->nr_events == 0)
277 return 0;
278
5f86b80b
JO
279 switch (how) {
280 case OE_FLUSH__FINAL:
281 oe->next_flush = ULLONG_MAX;
282 break;
283
284 case OE_FLUSH__HALF:
285 {
286 struct ordered_event *first, *last;
287 struct list_head *head = &oe->events;
288
289 first = list_entry(head->next, struct ordered_event, list);
290 last = oe->last;
291
292 /* Warn if we are called before any event got allocated. */
293 if (WARN_ONCE(!last || list_empty(head), "empty queue"))
294 return 0;
295
296 oe->next_flush = first->timestamp;
297 oe->next_flush += (last->timestamp - first->timestamp) / 2;
298 break;
299 }
300
301 case OE_FLUSH__ROUND:
b0a45203 302 case OE_FLUSH__NONE:
5f86b80b
JO
303 default:
304 break;
305 };
306
cee3ab9c
JO
307 pr_oe_time(oe->next_flush, "next_flush - ordered_events__flush PRE %s, nr_events %u\n",
308 str[how], oe->nr_events);
309 pr_oe_time(oe->max_timestamp, "max_timestamp\n");
310
b7b61cbe 311 err = __ordered_events__flush(oe);
5f86b80b
JO
312
313 if (!err) {
314 if (how == OE_FLUSH__ROUND)
315 oe->next_flush = oe->max_timestamp;
b0a45203
JO
316
317 oe->last_flush_type = how;
5f86b80b
JO
318 }
319
cee3ab9c
JO
320 pr_oe_time(oe->next_flush, "next_flush - ordered_events__flush POST %s, nr_events %u\n",
321 str[how], oe->nr_events);
322 pr_oe_time(oe->last_flush, "last_flush\n");
323
5f86b80b
JO
324 return err;
325}
36522f5c 326
9870d780 327void ordered_events__init(struct ordered_events *oe, ordered_events__deliver_t deliver)
36522f5c
JO
328{
329 INIT_LIST_HEAD(&oe->events);
330 INIT_LIST_HEAD(&oe->cache);
331 INIT_LIST_HEAD(&oe->to_free);
332 oe->max_alloc_size = (u64) -1;
333 oe->cur_alloc_size = 0;
d10eb1eb 334 oe->deliver = deliver;
36522f5c 335}
adc56ed1 336
d5ceb62b
JO
337static void
338ordered_events_buffer__free(struct ordered_events_buffer *buffer,
339 unsigned int max, struct ordered_events *oe)
340{
341 if (oe->copy_on_queue) {
342 unsigned int i;
343
344 for (i = 0; i < max; i++)
345 __free_dup_event(oe, buffer->event[i].event);
346 }
347
348 free(buffer);
349}
350
adc56ed1
JO
351void ordered_events__free(struct ordered_events *oe)
352{
d5ceb62b 353 struct ordered_events_buffer *buffer, *tmp;
adc56ed1 354
d5ceb62b
JO
355 if (list_empty(&oe->to_free))
356 return;
357
358 /*
359 * Current buffer might not have all the events allocated
360 * yet, we need to free only allocated ones ...
361 */
362 list_del(&oe->buffer->list);
363 ordered_events_buffer__free(oe->buffer, oe->buffer_idx, oe);
364
365 /* ... and continue with the rest */
366 list_for_each_entry_safe(buffer, tmp, &oe->to_free, list) {
367 list_del(&buffer->list);
368 ordered_events_buffer__free(buffer, MAX_SAMPLE_BUFFER, oe);
adc56ed1
JO
369 }
370}
4532f642
WN
371
372void ordered_events__reinit(struct ordered_events *oe)
373{
374 ordered_events__deliver_t old_deliver = oe->deliver;
375
376 ordered_events__free(oe);
377 memset(oe, '\0', sizeof(*oe));
378 ordered_events__init(oe, old_deliver);
379}