]>
Commit | Line | Data |
---|---|---|
00e5a55c BS |
1 | From: Peter Zijlstra <a.p.zijlstra@chello.nl> |
2 | Subject: mm: kmem_alloc_estimate() | |
3 | Patch-mainline: No | |
4 | References: FATE#303834 | |
5 | ||
6 | Provide a method to get the upper bound on the pages needed to allocate | |
7 | a given number of objects from a given kmem_cache. | |
8 | ||
9 | This lays the foundation for a generic reserve framework as presented in | |
10 | a later patch in this series. This framework needs to convert object demand | |
11 | (kmalloc() bytes, kmem_cache_alloc() objects) to pages. | |
12 | ||
13 | Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> | |
14 | Acked-by: Neil Brown <neilb@suse.de> | |
15 | Acked-by: Suresh Jayaraman <sjayaraman@suse.de> | |
16 | ||
17 | --- | |
18 | include/linux/slab.h | 4 ++ | |
19 | mm/slab.c | 75 +++++++++++++++++++++++++++++++++++++++++++ | |
20 | mm/slob.c | 67 +++++++++++++++++++++++++++++++++++++++ | |
21 | mm/slub.c | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++ | |
22 | 4 files changed, 233 insertions(+) | |
23 | ||
24 | --- a/include/linux/slab.h | |
25 | +++ b/include/linux/slab.h | |
26 | @@ -65,6 +65,8 @@ void kmem_cache_free(struct kmem_cache * | |
27 | unsigned int kmem_cache_size(struct kmem_cache *); | |
28 | const char *kmem_cache_name(struct kmem_cache *); | |
29 | int kmem_ptr_validate(struct kmem_cache *cachep, const void *ptr); | |
30 | +unsigned kmem_alloc_estimate(struct kmem_cache *cachep, | |
31 | + gfp_t flags, int objects); | |
32 | ||
33 | /* | |
34 | * Please use this macro to create slab caches. Simply specify the | |
35 | @@ -100,6 +102,8 @@ void * __must_check __krealloc(const voi | |
36 | void * __must_check krealloc(const void *, size_t, gfp_t); | |
37 | void kfree(const void *); | |
38 | size_t ksize(const void *); | |
39 | +unsigned kmalloc_estimate_objs(size_t, gfp_t, int); | |
40 | +unsigned kmalloc_estimate_bytes(gfp_t, size_t); | |
41 | ||
42 | /* | |
43 | * Allocator specific definitions. These are mainly used to establish optimized | |
44 | --- a/mm/slab.c | |
45 | +++ b/mm/slab.c | |
46 | @@ -3846,6 +3846,81 @@ const char *kmem_cache_name(struct kmem_ | |
47 | EXPORT_SYMBOL_GPL(kmem_cache_name); | |
48 | ||
49 | /* | |
50 | + * Calculate the upper bound of pages required to sequentially allocate | |
51 | + * @objects objects from @cachep. | |
52 | + */ | |
53 | +unsigned kmem_alloc_estimate(struct kmem_cache *cachep, | |
54 | + gfp_t flags, int objects) | |
55 | +{ | |
56 | + /* | |
57 | + * (1) memory for objects, | |
58 | + */ | |
59 | + unsigned nr_slabs = DIV_ROUND_UP(objects, cachep->num); | |
60 | + unsigned nr_pages = nr_slabs << cachep->gfporder; | |
61 | + | |
62 | + /* | |
63 | + * (2) memory for each per-cpu queue (nr_cpu_ids), | |
64 | + * (3) memory for each per-node alien queues (nr_cpu_ids), and | |
65 | + * (4) some amount of memory for the slab management structures | |
66 | + * | |
67 | + * XXX: truely account these | |
68 | + */ | |
69 | + nr_pages += 1 + ilog2(nr_pages); | |
70 | + | |
71 | + return nr_pages; | |
72 | +} | |
73 | + | |
74 | +/* | |
75 | + * Calculate the upper bound of pages required to sequentially allocate | |
76 | + * @count objects of @size bytes from kmalloc given @flags. | |
77 | + */ | |
78 | +unsigned kmalloc_estimate_objs(size_t size, gfp_t flags, int count) | |
79 | +{ | |
80 | + struct kmem_cache *s = kmem_find_general_cachep(size, flags); | |
81 | + if (!s) | |
82 | + return 0; | |
83 | + | |
84 | + return kmem_alloc_estimate(s, flags, count); | |
85 | +} | |
86 | +EXPORT_SYMBOL_GPL(kmalloc_estimate_objs); | |
87 | + | |
88 | +/* | |
89 | + * Calculate the upper bound of pages requires to sequentially allocate @bytes | |
90 | + * from kmalloc in an unspecified number of allocations of nonuniform size. | |
91 | + */ | |
92 | +unsigned kmalloc_estimate_bytes(gfp_t flags, size_t bytes) | |
93 | +{ | |
94 | + unsigned long pages; | |
95 | + struct cache_sizes *csizep = malloc_sizes; | |
96 | + | |
97 | + /* | |
98 | + * multiply by two, in order to account the worst case slack space | |
99 | + * due to the power-of-two allocation sizes. | |
100 | + */ | |
101 | + pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE); | |
102 | + | |
103 | + /* | |
104 | + * add the kmem_cache overhead of each possible kmalloc cache | |
105 | + */ | |
106 | + for (csizep = malloc_sizes; csizep->cs_cachep; csizep++) { | |
107 | + struct kmem_cache *s; | |
108 | + | |
109 | +#ifdef CONFIG_ZONE_DMA | |
110 | + if (unlikely(flags & __GFP_DMA)) | |
111 | + s = csizep->cs_dmacachep; | |
112 | + else | |
113 | +#endif | |
114 | + s = csizep->cs_cachep; | |
115 | + | |
116 | + if (s) | |
117 | + pages += kmem_alloc_estimate(s, flags, 0); | |
118 | + } | |
119 | + | |
120 | + return pages; | |
121 | +} | |
122 | +EXPORT_SYMBOL_GPL(kmalloc_estimate_bytes); | |
123 | + | |
124 | +/* | |
125 | * This initializes kmem_list3 or resizes various caches for all nodes. | |
126 | */ | |
127 | static int alloc_kmemlist(struct kmem_cache *cachep) | |
128 | --- a/mm/slob.c | |
129 | +++ b/mm/slob.c | |
130 | @@ -659,3 +659,70 @@ void __init kmem_cache_init(void) | |
131 | { | |
132 | slob_ready = 1; | |
133 | } | |
134 | + | |
135 | +static __slob_estimate(unsigned size, unsigned align, unsigned objects) | |
136 | +{ | |
137 | + unsigned nr_pages; | |
138 | + | |
139 | + size = SLOB_UNIT * SLOB_UNITS(size + align - 1); | |
140 | + | |
141 | + if (size <= PAGE_SIZE) { | |
142 | + nr_pages = DIV_ROUND_UP(objects, PAGE_SIZE / size); | |
143 | + } else { | |
144 | + nr_pages = objects << get_order(size); | |
145 | + } | |
146 | + | |
147 | + return nr_pages; | |
148 | +} | |
149 | + | |
150 | +/* | |
151 | + * Calculate the upper bound of pages required to sequentially allocate | |
152 | + * @objects objects from @cachep. | |
153 | + */ | |
154 | +unsigned kmem_alloc_estimate(struct kmem_cache *c, gfp_t flags, int objects) | |
155 | +{ | |
156 | + unsigned size = c->size; | |
157 | + | |
158 | + if (c->flags & SLAB_DESTROY_BY_RCU) | |
159 | + size += sizeof(struct slob_rcu); | |
160 | + | |
161 | + return __slob_estimate(size, c->align, objects); | |
162 | +} | |
163 | + | |
164 | +/* | |
165 | + * Calculate the upper bound of pages required to sequentially allocate | |
166 | + * @count objects of @size bytes from kmalloc given @flags. | |
167 | + */ | |
168 | +unsigned kmalloc_estimate_objs(size_t size, gfp_t flags, int count) | |
169 | +{ | |
170 | + unsigned align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); | |
171 | + | |
172 | + return __slob_estimate(size, align, count); | |
173 | +} | |
174 | +EXPORT_SYMBOL_GPL(kmalloc_estimate_objs); | |
175 | + | |
176 | +/* | |
177 | + * Calculate the upper bound of pages requires to sequentially allocate @bytes | |
178 | + * from kmalloc in an unspecified number of allocations of nonuniform size. | |
179 | + */ | |
180 | +unsigned kmalloc_estimate_bytes(gfp_t flags, size_t bytes) | |
181 | +{ | |
182 | + unsigned long pages; | |
183 | + | |
184 | + /* | |
185 | + * Multiply by two, in order to account the worst case slack space | |
186 | + * due to the power-of-two allocation sizes. | |
187 | + * | |
188 | + * While not true for slob, it cannot do worse than that for sequential | |
189 | + * allocations. | |
190 | + */ | |
191 | + pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE); | |
192 | + | |
193 | + /* | |
194 | + * Our power of two series starts at PAGE_SIZE, so add one page. | |
195 | + */ | |
196 | + pages++; | |
197 | + | |
198 | + return pages; | |
199 | +} | |
200 | +EXPORT_SYMBOL_GPL(kmalloc_estimate_bytes); | |
201 | --- a/mm/slub.c | |
202 | +++ b/mm/slub.c | |
203 | @@ -2399,6 +2399,42 @@ const char *kmem_cache_name(struct kmem_ | |
204 | } | |
205 | EXPORT_SYMBOL(kmem_cache_name); | |
206 | ||
207 | +/* | |
208 | + * Calculate the upper bound of pages required to sequentially allocate | |
209 | + * @objects objects from @cachep. | |
210 | + * | |
211 | + * We should use s->min_objects because those are the least efficient. | |
212 | + */ | |
213 | +unsigned kmem_alloc_estimate(struct kmem_cache *s, gfp_t flags, int objects) | |
214 | +{ | |
215 | + unsigned long pages; | |
216 | + struct kmem_cache_order_objects x; | |
217 | + | |
218 | + if (WARN_ON(!s) || WARN_ON(!oo_objects(s->min))) | |
219 | + return 0; | |
220 | + | |
221 | + x = s->min; | |
222 | + pages = DIV_ROUND_UP(objects, oo_objects(x)) << oo_order(x); | |
223 | + | |
224 | + /* | |
225 | + * Account the possible additional overhead if the slab holds more that | |
226 | + * one object. Use s->max_objects because that's the worst case. | |
227 | + */ | |
228 | + x = s->oo; | |
229 | + if (oo_objects(x) > 1) { | |
230 | + /* | |
231 | + * Account the possible additional overhead if per cpu slabs | |
232 | + * are currently empty and have to be allocated. This is very | |
233 | + * unlikely but a possible scenario immediately after | |
234 | + * kmem_cache_shrink. | |
235 | + */ | |
236 | + pages += num_possible_cpus() << oo_order(x); | |
237 | + } | |
238 | + | |
239 | + return pages; | |
240 | +} | |
241 | +EXPORT_SYMBOL_GPL(kmem_alloc_estimate); | |
242 | + | |
243 | static void list_slab_objects(struct kmem_cache *s, struct page *page, | |
244 | const char *text) | |
245 | { | |
246 | @@ -2776,6 +2812,57 @@ void kfree(const void *x) | |
247 | EXPORT_SYMBOL(kfree); | |
248 | ||
249 | /* | |
250 | + * Calculate the upper bound of pages required to sequentially allocate | |
251 | + * @count objects of @size bytes from kmalloc given @flags. | |
252 | + */ | |
253 | +unsigned kmalloc_estimate_objs(size_t size, gfp_t flags, int count) | |
254 | +{ | |
255 | + struct kmem_cache *s = get_slab(size, flags); | |
256 | + if (!s) | |
257 | + return 0; | |
258 | + | |
259 | + return kmem_alloc_estimate(s, flags, count); | |
260 | + | |
261 | +} | |
262 | +EXPORT_SYMBOL_GPL(kmalloc_estimate_objs); | |
263 | + | |
264 | +/* | |
265 | + * Calculate the upper bound of pages requires to sequentially allocate @bytes | |
266 | + * from kmalloc in an unspecified number of allocations of nonuniform size. | |
267 | + */ | |
268 | +unsigned kmalloc_estimate_bytes(gfp_t flags, size_t bytes) | |
269 | +{ | |
270 | + int i; | |
271 | + unsigned long pages; | |
272 | + | |
273 | + /* | |
274 | + * multiply by two, in order to account the worst case slack space | |
275 | + * due to the power-of-two allocation sizes. | |
276 | + */ | |
277 | + pages = DIV_ROUND_UP(2 * bytes, PAGE_SIZE); | |
278 | + | |
279 | + /* | |
280 | + * add the kmem_cache overhead of each possible kmalloc cache | |
281 | + */ | |
282 | + for (i = 1; i < PAGE_SHIFT; i++) { | |
283 | + struct kmem_cache *s; | |
284 | + | |
285 | +#ifdef CONFIG_ZONE_DMA | |
286 | + if (unlikely(flags & SLUB_DMA)) | |
287 | + s = dma_kmalloc_cache(i, flags); | |
288 | + else | |
289 | +#endif | |
290 | + s = &kmalloc_caches[i]; | |
291 | + | |
292 | + if (s) | |
293 | + pages += kmem_alloc_estimate(s, flags, 0); | |
294 | + } | |
295 | + | |
296 | + return pages; | |
297 | +} | |
298 | +EXPORT_SYMBOL_GPL(kmalloc_estimate_bytes); | |
299 | + | |
300 | +/* | |
301 | * kmem_cache_shrink removes empty slabs from the partial lists and sorts | |
302 | * the remaining slabs by the number of items in use. The slabs with the | |
303 | * most items in use come first. New allocations will then fill those up |