1 From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
2 Subject: LTTng instrumentation - hugetlb
5 LTTng instrumentation - hugetlb
7 Instrumentation of hugetlb activity (alloc/free/reserve/grab/release).
9 Those tracepoints are used by LTTng.
11 About the performance impact of tracepoints (which is comparable to markers),
12 even without immediate values optimizations, tests done by Hideo Aoki on ia64
13 show no regression. His test case was using hackbench on a kernel where
14 scheduler instrumentation (about 5 events in code scheduler code) was added.
15 See the "Tracepoints" patch header for performance result detail.
18 - instrument page grab, buddy allocator alloc, page release.
20 Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
21 CC: William Lee Irwin III <wli@holomorphy.com>
22 CC: Masami Hiramatsu <mhiramat@redhat.com>
23 CC: 'Peter Zijlstra' <peterz@infradead.org>
24 CC: "Frank Ch. Eigler" <fche@redhat.com>
25 CC: 'Ingo Molnar' <mingo@elte.hu>
26 CC: 'Hideo AOKI' <haoki@redhat.com>
27 CC: Takashi Nishiie <t-nishiie@np.css.fujitsu.com>
28 CC: 'Steven Rostedt' <rostedt@goodmis.org>
29 CC: Eduard - Gabriel Munteanu <eduard.munteanu@linux360.ro>
31 Acked-by: Jan Blunck <jblunck@suse.de>
34 include/trace/hugetlb.h | 28 ++++++++++++++++++++++++
35 mm/hugetlb.c | 56 +++++++++++++++++++++++++++++++++---------------
36 2 files changed, 67 insertions(+), 17 deletions(-)
39 +++ b/include/trace/hugetlb.h
41 +#ifndef _TRACE_HUGETLB_H
42 +#define _TRACE_HUGETLB_H
44 +#include <linux/tracepoint.h>
46 +DEFINE_TRACE(hugetlb_page_release,
47 + TPPROTO(struct page *page),
49 +DEFINE_TRACE(hugetlb_page_grab,
50 + TPPROTO(struct page *page),
52 +DEFINE_TRACE(hugetlb_buddy_pgalloc,
53 + TPPROTO(struct page *page),
55 +DEFINE_TRACE(hugetlb_page_alloc,
56 + TPPROTO(struct page *page),
58 +DEFINE_TRACE(hugetlb_page_free,
59 + TPPROTO(struct page *page),
61 +DEFINE_TRACE(hugetlb_pages_reserve,
62 + TPPROTO(struct inode *inode, long from, long to, int ret),
63 + TPARGS(inode, from, to, ret));
64 +DEFINE_TRACE(hugetlb_pages_unreserve,
65 + TPPROTO(struct inode *inode, long offset, long freed),
66 + TPARGS(inode, offset, freed));
72 #include <linux/mutex.h>
73 #include <linux/bootmem.h>
74 #include <linux/sysfs.h>
75 +#include <trace/hugetlb.h>
78 #include <asm/pgtable.h>
79 @@ -492,6 +493,7 @@ static void update_and_free_page(struct
81 VM_BUG_ON(h->order >= MAX_ORDER);
83 + trace_hugetlb_page_release(page);
85 h->nr_huge_pages_node[page_to_nid(page)]--;
86 for (i = 0; i < pages_per_huge_page(h); i++) {
87 @@ -526,6 +528,7 @@ static void free_huge_page(struct page *
88 int nid = page_to_nid(page);
89 struct address_space *mapping;
91 + trace_hugetlb_page_free(page);
92 mapping = (struct address_space *) page_private(page);
93 set_page_private(page, 0);
94 BUG_ON(page_count(page));
95 @@ -593,8 +596,10 @@ static struct page *alloc_fresh_huge_pag
99 - if (h->order >= MAX_ORDER)
101 + if (h->order >= MAX_ORDER) {
106 page = alloc_pages_node(nid,
107 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
108 @@ -603,11 +608,13 @@ static struct page *alloc_fresh_huge_pag
110 if (arch_prepare_hugepage(page)) {
111 __free_pages(page, huge_page_order(h));
116 prep_new_huge_page(h, page, nid);
120 + trace_hugetlb_page_grab(page);
124 @@ -691,7 +698,8 @@ static struct page *alloc_buddy_huge_pag
125 spin_lock(&hugetlb_lock);
126 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
127 spin_unlock(&hugetlb_lock);
133 h->surplus_huge_pages++;
134 @@ -729,7 +737,8 @@ static struct page *alloc_buddy_huge_pag
135 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
137 spin_unlock(&hugetlb_lock);
140 + trace_hugetlb_buddy_pgalloc(page);
144 @@ -968,6 +977,7 @@ static struct page *alloc_huge_page(stru
146 vma_commit_reservation(h, vma, addr);
148 + trace_hugetlb_page_alloc(page);
152 @@ -2233,11 +2243,12 @@ int hugetlb_reserve_pages(struct inode *
154 struct vm_area_struct *vma)
159 struct hstate *h = hstate_inode(inode);
161 if (vma && vma->vm_flags & VM_NORESERVE)
166 * Shared mappings base their reservation on the number of pages that
167 @@ -2249,8 +2260,10 @@ int hugetlb_reserve_pages(struct inode *
168 chg = region_chg(&inode->i_mapping->private_list, from, to);
170 struct resv_map *resv_map = resv_map_alloc();
180 @@ -2258,25 +2271,34 @@ int hugetlb_reserve_pages(struct inode *
181 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
191 - if (hugetlb_get_quota(inode->i_mapping, chg))
193 + if (hugetlb_get_quota(inode->i_mapping, chg)) {
197 ret = hugetlb_acct_memory(h, chg);
199 hugetlb_put_quota(inode->i_mapping, chg);
203 if (!vma || vma->vm_flags & VM_MAYSHARE)
204 region_add(&inode->i_mapping->private_list, from, to);
207 + trace_hugetlb_pages_reserve(inode, from, to, ret);
211 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
213 struct hstate *h = hstate_inode(inode);
214 - long chg = region_truncate(&inode->i_mapping->private_list, offset);
217 + trace_hugetlb_pages_unreserve(inode, offset, freed);
218 + chg = region_truncate(&inode->i_mapping->private_list, offset);
220 spin_lock(&inode->i_lock);
221 inode->i_blocks -= blocks_per_huge_page(h);