2 * Performance events ring-buffer code:
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
12 #include <linux/perf_event.h>
13 #include <linux/vmalloc.h>
14 #include <linux/slab.h>
15 #include <linux/circ_buf.h>
16 #include <linux/poll.h>
17 #include <linux/nospec.h>
21 static void perf_output_wakeup(struct perf_output_handle
*handle
)
23 atomic_set(&handle
->rb
->poll
, EPOLLIN
);
25 handle
->event
->pending_wakeup
= 1;
26 irq_work_queue(&handle
->event
->pending
);
30 * We need to ensure a later event_id doesn't publish a head when a former
31 * event isn't done writing. However since we need to deal with NMIs we
32 * cannot fully serialize things.
34 * We only publish the head (and generate a wakeup) when the outer-most
37 static void perf_output_get_handle(struct perf_output_handle
*handle
)
39 struct ring_buffer
*rb
= handle
->rb
;
43 handle
->wakeup
= local_read(&rb
->wakeup
);
46 static void perf_output_put_handle(struct perf_output_handle
*handle
)
48 struct ring_buffer
*rb
= handle
->rb
;
52 head
= local_read(&rb
->head
);
55 * IRQ/NMI can happen here and advance @rb->head, causing our
56 * load above to be stale.
60 * If this isn't the outermost nesting, we don't have to update
61 * @rb->user_page->data_head.
63 if (local_read(&rb
->nest
) > 1) {
69 * Since the mmap() consumer (userspace) can run on a different CPU:
73 * if (LOAD ->data_tail) { LOAD ->data_head
75 * STORE $data LOAD $data
76 * smp_wmb() (B) smp_mb() (D)
77 * STORE ->data_head STORE ->data_tail
80 * Where A pairs with D, and B pairs with C.
82 * In our case (A) is a control dependency that separates the load of
83 * the ->data_tail and the stores of $data. In case ->data_tail
84 * indicates there is no room in the buffer to store $data we do not.
86 * D needs to be a full barrier since it separates the data READ
87 * from the tail WRITE.
89 * For B a WMB is sufficient since it separates two WRITEs, and for C
90 * an RMB is sufficient since it separates two READs.
92 * See perf_output_begin().
94 smp_wmb(); /* B, matches C */
95 rb
->user_page
->data_head
= head
;
98 * We must publish the head before decrementing the nest count,
99 * otherwise an IRQ/NMI can publish a more recent head value and our
100 * write will (temporarily) publish a stale value.
103 local_set(&rb
->nest
, 0);
106 * Ensure we decrement @rb->nest before we validate the @rb->head.
107 * Otherwise we cannot be sure we caught the 'last' nested update.
110 if (unlikely(head
!= local_read(&rb
->head
))) {
111 local_inc(&rb
->nest
);
115 if (handle
->wakeup
!= local_read(&rb
->wakeup
))
116 perf_output_wakeup(handle
);
122 static __always_inline
bool
123 ring_buffer_has_space(unsigned long head
, unsigned long tail
,
124 unsigned long data_size
, unsigned int size
,
128 return CIRC_SPACE(head
, tail
, data_size
) >= size
;
130 return CIRC_SPACE(tail
, head
, data_size
) >= size
;
133 static __always_inline
int
134 __perf_output_begin(struct perf_output_handle
*handle
,
135 struct perf_event
*event
, unsigned int size
,
138 struct ring_buffer
*rb
;
139 unsigned long tail
, offset
, head
;
140 int have_lost
, page_shift
;
142 struct perf_event_header header
;
149 * For inherited events we send all the output towards the parent.
152 event
= event
->parent
;
154 rb
= rcu_dereference(event
->rb
);
158 if (unlikely(rb
->paused
)) {
160 local_inc(&rb
->lost
);
165 handle
->event
= event
;
167 have_lost
= local_read(&rb
->lost
);
168 if (unlikely(have_lost
)) {
169 size
+= sizeof(lost_event
);
170 if (event
->attr
.sample_id_all
)
171 size
+= event
->id_header_size
;
174 perf_output_get_handle(handle
);
177 tail
= READ_ONCE(rb
->user_page
->data_tail
);
178 offset
= head
= local_read(&rb
->head
);
179 if (!rb
->overwrite
) {
180 if (unlikely(!ring_buffer_has_space(head
, tail
,
187 * The above forms a control dependency barrier separating the
188 * @tail load above from the data stores below. Since the @tail
189 * load is required to compute the branch to fail below.
191 * A, matches D; the full memory barrier userspace SHOULD issue
192 * after reading the data and before storing the new tail
195 * See perf_output_put_handle().
202 } while (local_cmpxchg(&rb
->head
, offset
, head
) != offset
);
210 * We rely on the implied barrier() by local_cmpxchg() to ensure
211 * none of the data stores below can be lifted up by the compiler.
214 if (unlikely(head
- local_read(&rb
->wakeup
) > rb
->watermark
))
215 local_add(rb
->watermark
, &rb
->wakeup
);
217 page_shift
= PAGE_SHIFT
+ page_order(rb
);
219 handle
->page
= (offset
>> page_shift
) & (rb
->nr_pages
- 1);
220 offset
&= (1UL << page_shift
) - 1;
221 handle
->addr
= rb
->data_pages
[handle
->page
] + offset
;
222 handle
->size
= (1UL << page_shift
) - offset
;
224 if (unlikely(have_lost
)) {
225 struct perf_sample_data sample_data
;
227 lost_event
.header
.size
= sizeof(lost_event
);
228 lost_event
.header
.type
= PERF_RECORD_LOST
;
229 lost_event
.header
.misc
= 0;
230 lost_event
.id
= event
->id
;
231 lost_event
.lost
= local_xchg(&rb
->lost
, 0);
233 perf_event_header__init_id(&lost_event
.header
,
234 &sample_data
, event
);
235 perf_output_put(handle
, lost_event
);
236 perf_event__output_id_sample(event
, handle
, &sample_data
);
242 local_inc(&rb
->lost
);
243 perf_output_put_handle(handle
);
250 int perf_output_begin_forward(struct perf_output_handle
*handle
,
251 struct perf_event
*event
, unsigned int size
)
253 return __perf_output_begin(handle
, event
, size
, false);
256 int perf_output_begin_backward(struct perf_output_handle
*handle
,
257 struct perf_event
*event
, unsigned int size
)
259 return __perf_output_begin(handle
, event
, size
, true);
262 int perf_output_begin(struct perf_output_handle
*handle
,
263 struct perf_event
*event
, unsigned int size
)
266 return __perf_output_begin(handle
, event
, size
,
267 unlikely(is_write_backward(event
)));
270 unsigned int perf_output_copy(struct perf_output_handle
*handle
,
271 const void *buf
, unsigned int len
)
273 return __output_copy(handle
, buf
, len
);
276 unsigned int perf_output_skip(struct perf_output_handle
*handle
,
279 return __output_skip(handle
, NULL
, len
);
282 void perf_output_end(struct perf_output_handle
*handle
)
284 perf_output_put_handle(handle
);
289 ring_buffer_init(struct ring_buffer
*rb
, long watermark
, int flags
)
291 long max_size
= perf_data_size(rb
);
294 rb
->watermark
= min(max_size
, watermark
);
297 rb
->watermark
= max_size
/ 2;
299 if (flags
& RING_BUFFER_WRITABLE
)
304 atomic_set(&rb
->refcount
, 1);
306 INIT_LIST_HEAD(&rb
->event_list
);
307 spin_lock_init(&rb
->event_lock
);
310 * perf_output_begin() only checks rb->paused, therefore
311 * rb->paused must be true if we have no pages for output.
317 void perf_aux_output_flag(struct perf_output_handle
*handle
, u64 flags
)
320 * OVERWRITE is determined by perf_aux_output_end() and can't
321 * be passed in directly.
323 if (WARN_ON_ONCE(flags
& PERF_AUX_FLAG_OVERWRITE
))
326 handle
->aux_flags
|= flags
;
328 EXPORT_SYMBOL_GPL(perf_aux_output_flag
);
331 * This is called before hardware starts writing to the AUX area to
332 * obtain an output handle and make sure there's room in the buffer.
333 * When the capture completes, call perf_aux_output_end() to commit
334 * the recorded data to the buffer.
336 * The ordering is similar to that of perf_output_{begin,end}, with
337 * the exception of (B), which should be taken care of by the pmu
338 * driver, since ordering rules will differ depending on hardware.
340 * Call this from pmu::start(); see the comment in perf_aux_output_end()
341 * about its use in pmu callbacks. Both can also be called from the PMI
344 void *perf_aux_output_begin(struct perf_output_handle
*handle
,
345 struct perf_event
*event
)
347 struct perf_event
*output_event
= event
;
348 unsigned long aux_head
, aux_tail
;
349 struct ring_buffer
*rb
;
351 if (output_event
->parent
)
352 output_event
= output_event
->parent
;
355 * Since this will typically be open across pmu::add/pmu::del, we
356 * grab ring_buffer's refcount instead of holding rcu read lock
357 * to make sure it doesn't disappear under us.
359 rb
= ring_buffer_get(output_event
);
367 * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),
368 * about to get freed, so we leave immediately.
370 * Checking rb::aux_mmap_count and rb::refcount has to be done in
371 * the same order, see perf_mmap_close. Otherwise we end up freeing
372 * aux pages in this path, which is a bug, because in_atomic().
374 if (!atomic_read(&rb
->aux_mmap_count
))
377 if (!atomic_inc_not_zero(&rb
->aux_refcount
))
381 * Nesting is not supported for AUX area, make sure nested
382 * writers are caught early
384 if (WARN_ON_ONCE(local_xchg(&rb
->aux_nest
, 1)))
387 aux_head
= rb
->aux_head
;
390 handle
->event
= event
;
391 handle
->head
= aux_head
;
393 handle
->aux_flags
= 0;
396 * In overwrite mode, AUX data stores do not depend on aux_tail,
397 * therefore (A) control dependency barrier does not exist. The
398 * (B) <-> (C) ordering is still observed by the pmu driver.
400 if (!rb
->aux_overwrite
) {
401 aux_tail
= READ_ONCE(rb
->user_page
->aux_tail
);
402 handle
->wakeup
= rb
->aux_wakeup
+ rb
->aux_watermark
;
403 if (aux_head
- aux_tail
< perf_aux_size(rb
))
404 handle
->size
= CIRC_SPACE(aux_head
, aux_tail
, perf_aux_size(rb
));
407 * handle->size computation depends on aux_tail load; this forms a
408 * control dependency barrier separating aux_tail load from aux data
409 * store that will be enabled on successful return
411 if (!handle
->size
) { /* A, matches D */
412 event
->pending_disable
= smp_processor_id();
413 perf_output_wakeup(handle
);
414 local_set(&rb
->aux_nest
, 0);
419 return handle
->rb
->aux_priv
;
427 handle
->event
= NULL
;
431 EXPORT_SYMBOL_GPL(perf_aux_output_begin
);
433 static __always_inline
bool rb_need_aux_wakeup(struct ring_buffer
*rb
)
435 if (rb
->aux_overwrite
)
438 if (rb
->aux_head
- rb
->aux_wakeup
>= rb
->aux_watermark
) {
439 rb
->aux_wakeup
= rounddown(rb
->aux_head
, rb
->aux_watermark
);
447 * Commit the data written by hardware into the ring buffer by adjusting
448 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
449 * pmu driver's responsibility to observe ordering rules of the hardware,
450 * so that all the data is externally visible before this is called.
452 * Note: this has to be called from pmu::stop() callback, as the assumption
453 * of the AUX buffer management code is that after pmu::stop(), the AUX
454 * transaction must be stopped and therefore drop the AUX reference count.
456 void perf_aux_output_end(struct perf_output_handle
*handle
, unsigned long size
)
458 bool wakeup
= !!(handle
->aux_flags
& PERF_AUX_FLAG_TRUNCATED
);
459 struct ring_buffer
*rb
= handle
->rb
;
460 unsigned long aux_head
;
462 /* in overwrite mode, driver provides aux_head via handle */
463 if (rb
->aux_overwrite
) {
464 handle
->aux_flags
|= PERF_AUX_FLAG_OVERWRITE
;
466 aux_head
= handle
->head
;
467 rb
->aux_head
= aux_head
;
469 handle
->aux_flags
&= ~PERF_AUX_FLAG_OVERWRITE
;
471 aux_head
= rb
->aux_head
;
472 rb
->aux_head
+= size
;
475 if (size
|| handle
->aux_flags
) {
477 * Only send RECORD_AUX if we have something useful to communicate
480 perf_event_aux_event(handle
->event
, aux_head
, size
,
484 rb
->user_page
->aux_head
= rb
->aux_head
;
485 if (rb_need_aux_wakeup(rb
))
489 if (handle
->aux_flags
& PERF_AUX_FLAG_TRUNCATED
)
490 handle
->event
->pending_disable
= smp_processor_id();
491 perf_output_wakeup(handle
);
494 handle
->event
= NULL
;
496 local_set(&rb
->aux_nest
, 0);
501 EXPORT_SYMBOL_GPL(perf_aux_output_end
);
504 * Skip over a given number of bytes in the AUX buffer, due to, for example,
505 * hardware's alignment constraints.
507 int perf_aux_output_skip(struct perf_output_handle
*handle
, unsigned long size
)
509 struct ring_buffer
*rb
= handle
->rb
;
511 if (size
> handle
->size
)
514 rb
->aux_head
+= size
;
516 rb
->user_page
->aux_head
= rb
->aux_head
;
517 if (rb_need_aux_wakeup(rb
)) {
518 perf_output_wakeup(handle
);
519 handle
->wakeup
= rb
->aux_wakeup
+ rb
->aux_watermark
;
522 handle
->head
= rb
->aux_head
;
523 handle
->size
-= size
;
527 EXPORT_SYMBOL_GPL(perf_aux_output_skip
);
529 void *perf_get_aux(struct perf_output_handle
*handle
)
531 /* this is only valid between perf_aux_output_begin and *_end */
535 return handle
->rb
->aux_priv
;
537 EXPORT_SYMBOL_GPL(perf_get_aux
);
539 #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
541 static struct page
*rb_alloc_aux_page(int node
, int order
)
545 if (order
> MAX_ORDER
)
549 page
= alloc_pages_node(node
, PERF_AUX_GFP
, order
);
550 } while (!page
&& order
--);
554 * Communicate the allocation size to the driver:
555 * if we managed to secure a high-order allocation,
556 * set its first page's private to this order;
557 * !PagePrivate(page) means it's just a normal page.
559 split_page(page
, order
);
560 SetPagePrivate(page
);
561 set_page_private(page
, order
);
567 static void rb_free_aux_page(struct ring_buffer
*rb
, int idx
)
569 struct page
*page
= virt_to_page(rb
->aux_pages
[idx
]);
571 ClearPagePrivate(page
);
572 page
->mapping
= NULL
;
576 static void __rb_free_aux(struct ring_buffer
*rb
)
581 * Should never happen, the last reference should be dropped from
582 * perf_mmap_close() path, which first stops aux transactions (which
583 * in turn are the atomic holders of aux_refcount) and then does the
584 * last rb_free_aux().
586 WARN_ON_ONCE(in_atomic());
589 rb
->free_aux(rb
->aux_priv
);
594 if (rb
->aux_nr_pages
) {
595 for (pg
= 0; pg
< rb
->aux_nr_pages
; pg
++)
596 rb_free_aux_page(rb
, pg
);
598 kfree(rb
->aux_pages
);
599 rb
->aux_nr_pages
= 0;
603 int rb_alloc_aux(struct ring_buffer
*rb
, struct perf_event
*event
,
604 pgoff_t pgoff
, int nr_pages
, long watermark
, int flags
)
606 bool overwrite
= !(flags
& RING_BUFFER_WRITABLE
);
607 int node
= (event
->cpu
== -1) ? -1 : cpu_to_node(event
->cpu
);
608 int ret
= -ENOMEM
, max_order
= 0;
613 if (event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) {
615 * We need to start with the max_order that fits in nr_pages,
616 * not the other way around, hence ilog2() and not get_order.
618 max_order
= ilog2(nr_pages
);
621 * PMU requests more than one contiguous chunks of memory
622 * for SW double buffering
624 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_SW_DOUBLEBUF
) &&
633 rb
->aux_pages
= kcalloc_node(nr_pages
, sizeof(void *), GFP_KERNEL
,
638 rb
->free_aux
= event
->pmu
->free_aux
;
639 for (rb
->aux_nr_pages
= 0; rb
->aux_nr_pages
< nr_pages
;) {
643 order
= min(max_order
, ilog2(nr_pages
- rb
->aux_nr_pages
));
644 page
= rb_alloc_aux_page(node
, order
);
648 for (last
= rb
->aux_nr_pages
+ (1 << page_private(page
));
649 last
> rb
->aux_nr_pages
; rb
->aux_nr_pages
++)
650 rb
->aux_pages
[rb
->aux_nr_pages
] = page_address(page
++);
654 * In overwrite mode, PMUs that don't support SG may not handle more
655 * than one contiguous allocation, since they rely on PMI to do double
656 * buffering. In this case, the entire buffer has to be one contiguous
659 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) &&
661 struct page
*page
= virt_to_page(rb
->aux_pages
[0]);
663 if (page_private(page
) != max_order
)
667 rb
->aux_priv
= event
->pmu
->setup_aux(event
, rb
->aux_pages
, nr_pages
,
675 * aux_pages (and pmu driver's private data, aux_priv) will be
676 * referenced in both producer's and consumer's contexts, thus
677 * we keep a refcount here to make sure either of the two can
678 * reference them safely.
680 atomic_set(&rb
->aux_refcount
, 1);
682 rb
->aux_overwrite
= overwrite
;
683 rb
->aux_watermark
= watermark
;
685 if (!rb
->aux_watermark
&& !rb
->aux_overwrite
)
686 rb
->aux_watermark
= nr_pages
<< (PAGE_SHIFT
- 1);
690 rb
->aux_pgoff
= pgoff
;
697 void rb_free_aux(struct ring_buffer
*rb
)
699 if (atomic_dec_and_test(&rb
->aux_refcount
))
703 #ifndef CONFIG_PERF_USE_VMALLOC
706 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
710 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
712 if (pgoff
> rb
->nr_pages
)
716 return virt_to_page(rb
->user_page
);
718 return virt_to_page(rb
->data_pages
[pgoff
- 1]);
721 static void *perf_mmap_alloc_page(int cpu
)
726 node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
727 page
= alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
, 0);
731 return page_address(page
);
734 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
736 struct ring_buffer
*rb
;
740 size
= sizeof(struct ring_buffer
);
741 size
+= nr_pages
* sizeof(void *);
743 if (order_base_2(size
) >= PAGE_SHIFT
+MAX_ORDER
)
746 rb
= kzalloc(size
, GFP_KERNEL
);
750 rb
->user_page
= perf_mmap_alloc_page(cpu
);
754 for (i
= 0; i
< nr_pages
; i
++) {
755 rb
->data_pages
[i
] = perf_mmap_alloc_page(cpu
);
756 if (!rb
->data_pages
[i
])
757 goto fail_data_pages
;
760 rb
->nr_pages
= nr_pages
;
762 ring_buffer_init(rb
, watermark
, flags
);
767 for (i
--; i
>= 0; i
--)
768 free_page((unsigned long)rb
->data_pages
[i
]);
770 free_page((unsigned long)rb
->user_page
);
779 static void perf_mmap_free_page(unsigned long addr
)
781 struct page
*page
= virt_to_page((void *)addr
);
783 page
->mapping
= NULL
;
787 void rb_free(struct ring_buffer
*rb
)
791 perf_mmap_free_page((unsigned long)rb
->user_page
);
792 for (i
= 0; i
< rb
->nr_pages
; i
++)
793 perf_mmap_free_page((unsigned long)rb
->data_pages
[i
]);
798 static int data_page_nr(struct ring_buffer
*rb
)
800 return rb
->nr_pages
<< page_order(rb
);
804 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
806 /* The '>' counts in the user page. */
807 if (pgoff
> data_page_nr(rb
))
810 return vmalloc_to_page((void *)rb
->user_page
+ pgoff
* PAGE_SIZE
);
813 static void perf_mmap_unmark_page(void *addr
)
815 struct page
*page
= vmalloc_to_page(addr
);
817 page
->mapping
= NULL
;
820 static void rb_free_work(struct work_struct
*work
)
822 struct ring_buffer
*rb
;
826 rb
= container_of(work
, struct ring_buffer
, work
);
827 nr
= data_page_nr(rb
);
829 base
= rb
->user_page
;
830 /* The '<=' counts in the user page. */
831 for (i
= 0; i
<= nr
; i
++)
832 perf_mmap_unmark_page(base
+ (i
* PAGE_SIZE
));
838 void rb_free(struct ring_buffer
*rb
)
840 schedule_work(&rb
->work
);
843 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
845 struct ring_buffer
*rb
;
849 size
= sizeof(struct ring_buffer
);
850 size
+= sizeof(void *);
852 rb
= kzalloc(size
, GFP_KERNEL
);
856 INIT_WORK(&rb
->work
, rb_free_work
);
858 all_buf
= vmalloc_user((nr_pages
+ 1) * PAGE_SIZE
);
862 rb
->user_page
= all_buf
;
863 rb
->data_pages
[0] = all_buf
+ PAGE_SIZE
;
866 rb
->page_order
= ilog2(nr_pages
);
869 ring_buffer_init(rb
, watermark
, flags
);
883 perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
885 if (rb
->aux_nr_pages
) {
886 /* above AUX space */
887 if (pgoff
> rb
->aux_pgoff
+ rb
->aux_nr_pages
)
891 if (pgoff
>= rb
->aux_pgoff
) {
892 int aux_pgoff
= array_index_nospec(pgoff
- rb
->aux_pgoff
, rb
->aux_nr_pages
);
893 return virt_to_page(rb
->aux_pages
[aux_pgoff
]);
897 return __perf_mmap_to_page(rb
, pgoff
);