1 // SPDX-License-Identifier: GPL-2.0
4 * Test module for stress and analyze performance of vmalloc allocator.
5 * (C) 2018 Uladzislau Rezki (Sony) <urezki@gmail.com>
7 #include <linux/init.h>
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/vmalloc.h>
11 #include <linux/random.h>
12 #include <linux/kthread.h>
13 #include <linux/moduleparam.h>
14 #include <linux/completion.h>
15 #include <linux/delay.h>
16 #include <linux/rwsem.h>
18 #include <linux/rcupdate.h>
19 #include <linux/slab.h>
21 #define __param(type, name, init, msg) \
22 static type name = init; \
23 module_param(name, type, 0444); \
24 MODULE_PARM_DESC(name, msg) \
26 __param(int, nr_threads, 0,
27 "Number of workers to perform tests(min: 1 max: USHRT_MAX)");
29 __param(bool, sequential_test_order
, false,
30 "Use sequential stress tests order");
32 __param(int, test_repeat_count
, 1,
33 "Set test repeat counter");
35 __param(int, test_loop_count
, 1000000,
36 "Set test loop counter");
38 __param(int, nr_pages
, 0,
39 "Set number of pages for fix_size_alloc_test(default: 1)");
41 __param(int, run_test_mask
, INT_MAX
,
42 "Set tests specified in the mask.\n\n"
43 "\t\tid: 1, name: fix_size_alloc_test\n"
44 "\t\tid: 2, name: full_fit_alloc_test\n"
45 "\t\tid: 4, name: long_busy_list_alloc_test\n"
46 "\t\tid: 8, name: random_size_alloc_test\n"
47 "\t\tid: 16, name: fix_align_alloc_test\n"
48 "\t\tid: 32, name: random_size_align_alloc_test\n"
49 "\t\tid: 64, name: align_shift_alloc_test\n"
50 "\t\tid: 128, name: pcpu_alloc_test\n"
51 "\t\tid: 256, name: kvfree_rcu_1_arg_vmalloc_test\n"
52 "\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n"
53 /* Add a new test case description here. */
57 * Read write semaphore for synchronization of setup
58 * phase that is done in main thread and workers.
60 static DECLARE_RWSEM(prepare_for_test_rwsem
);
63 * Completion tracking for worker threads.
65 static DECLARE_COMPLETION(test_all_done_comp
);
66 static atomic_t test_n_undone
= ATOMIC_INIT(0);
69 test_report_one_done(void)
71 if (atomic_dec_and_test(&test_n_undone
))
72 complete(&test_all_done_comp
);
75 static int random_size_align_alloc_test(void)
77 unsigned long size
, align
;
82 for (i
= 0; i
< test_loop_count
; i
++) {
86 * Maximum 1024 pages, if PAGE_SIZE is 4096.
88 align
= 1 << (rnd
% 23);
93 size
= ((rnd
% 10) + 1) * PAGE_SIZE
;
95 ptr
= __vmalloc_node(size
, align
, GFP_KERNEL
| __GFP_ZERO
, 0,
96 __builtin_return_address(0));
107 * This test case is supposed to be failed.
109 static int align_shift_alloc_test(void)
115 for (i
= 0; i
< BITS_PER_LONG
; i
++) {
116 align
= ((unsigned long) 1) << i
;
118 ptr
= __vmalloc_node(PAGE_SIZE
, align
, GFP_KERNEL
|__GFP_ZERO
, 0,
119 __builtin_return_address(0));
129 static int fix_align_alloc_test(void)
134 for (i
= 0; i
< test_loop_count
; i
++) {
135 ptr
= __vmalloc_node(5 * PAGE_SIZE
, THREAD_ALIGN
<< 1,
136 GFP_KERNEL
| __GFP_ZERO
, 0,
137 __builtin_return_address(0));
147 static int random_size_alloc_test(void)
153 for (i
= 0; i
< test_loop_count
; i
++) {
157 p
= vmalloc(n
* PAGE_SIZE
);
169 static int long_busy_list_alloc_test(void)
176 ptr
= vmalloc(sizeof(void *) * 15000);
180 for (i
= 0; i
< 15000; i
++)
181 ptr
[i
] = vmalloc(1 * PAGE_SIZE
);
183 for (i
= 0; i
< test_loop_count
; i
++) {
184 ptr_1
= vmalloc(100 * PAGE_SIZE
);
188 ptr_2
= vmalloc(1 * PAGE_SIZE
);
194 *((__u8
*)ptr_1
) = 0;
195 *((__u8
*)ptr_2
) = 1;
205 for (i
= 0; i
< 15000; i
++)
212 static int full_fit_alloc_test(void)
214 void **ptr
, **junk_ptr
, *tmp
;
219 junk_length
= fls(num_online_cpus());
220 junk_length
*= (32 * 1024 * 1024 / PAGE_SIZE
);
222 ptr
= vmalloc(sizeof(void *) * junk_length
);
226 junk_ptr
= vmalloc(sizeof(void *) * junk_length
);
232 for (i
= 0; i
< junk_length
; i
++) {
233 ptr
[i
] = vmalloc(1 * PAGE_SIZE
);
234 junk_ptr
[i
] = vmalloc(1 * PAGE_SIZE
);
237 for (i
= 0; i
< junk_length
; i
++)
240 for (i
= 0; i
< test_loop_count
; i
++) {
241 tmp
= vmalloc(1 * PAGE_SIZE
);
254 for (i
= 0; i
< junk_length
; i
++)
263 static int fix_size_alloc_test(void)
268 for (i
= 0; i
< test_loop_count
; i
++) {
269 ptr
= vmalloc((nr_pages
> 0 ? nr_pages
:1) * PAGE_SIZE
);
283 pcpu_alloc_test(void)
286 #ifndef CONFIG_NEED_PER_CPU_KM
287 void __percpu
**pcpu
;
291 pcpu
= vmalloc(sizeof(void __percpu
*) * 35000);
295 for (i
= 0; i
< 35000; i
++) {
299 size
= (r
% (PAGE_SIZE
/ 4)) + 1;
305 align
= 1 << ((r
% 11) + 1);
307 pcpu
[i
] = __alloc_percpu(size
, align
);
312 for (i
= 0; i
< 35000; i
++)
313 free_percpu(pcpu
[i
]);
320 struct test_kvfree_rcu
{
322 unsigned char array
[20];
326 kvfree_rcu_1_arg_vmalloc_test(void)
328 struct test_kvfree_rcu
*p
;
331 for (i
= 0; i
< test_loop_count
; i
++) {
332 p
= vmalloc(1 * PAGE_SIZE
);
344 kvfree_rcu_2_arg_vmalloc_test(void)
346 struct test_kvfree_rcu
*p
;
349 for (i
= 0; i
< test_loop_count
; i
++) {
350 p
= vmalloc(1 * PAGE_SIZE
);
361 struct test_case_desc
{
362 const char *test_name
;
363 int (*test_func
)(void);
366 static struct test_case_desc test_case_array
[] = {
367 { "fix_size_alloc_test", fix_size_alloc_test
},
368 { "full_fit_alloc_test", full_fit_alloc_test
},
369 { "long_busy_list_alloc_test", long_busy_list_alloc_test
},
370 { "random_size_alloc_test", random_size_alloc_test
},
371 { "fix_align_alloc_test", fix_align_alloc_test
},
372 { "random_size_align_alloc_test", random_size_align_alloc_test
},
373 { "align_shift_alloc_test", align_shift_alloc_test
},
374 { "pcpu_alloc_test", pcpu_alloc_test
},
375 { "kvfree_rcu_1_arg_vmalloc_test", kvfree_rcu_1_arg_vmalloc_test
},
376 { "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test
},
377 /* Add a new test case here. */
380 struct test_case_data
{
386 static struct test_driver
{
387 struct task_struct
*task
;
388 struct test_case_data data
[ARRAY_SIZE(test_case_array
)];
394 static void shuffle_array(int *arr
, int n
)
399 for (i
= n
- 1; i
> 0; i
--) {
406 swap(arr
[i
], arr
[j
]);
410 static int test_func(void *private)
412 struct test_driver
*t
= private;
413 int random_array
[ARRAY_SIZE(test_case_array
)];
418 for (i
= 0; i
< ARRAY_SIZE(test_case_array
); i
++)
421 if (!sequential_test_order
)
422 shuffle_array(random_array
, ARRAY_SIZE(test_case_array
));
425 * Block until initialization is done.
427 down_read(&prepare_for_test_rwsem
);
429 t
->start
= get_cycles();
430 for (i
= 0; i
< ARRAY_SIZE(test_case_array
); i
++) {
431 index
= random_array
[i
];
434 * Skip tests if run_test_mask has been specified.
436 if (!((run_test_mask
& (1 << index
)) >> index
))
440 for (j
= 0; j
< test_repeat_count
; j
++) {
441 if (!test_case_array
[index
].test_func())
442 t
->data
[index
].test_passed
++;
444 t
->data
[index
].test_failed
++;
448 * Take an average time that test took.
450 delta
= (u64
) ktime_us_delta(ktime_get(), kt
);
451 do_div(delta
, (u32
) test_repeat_count
);
453 t
->data
[index
].time
= delta
;
455 t
->stop
= get_cycles();
457 up_read(&prepare_for_test_rwsem
);
458 test_report_one_done();
461 * Wait for the kthread_stop() call.
463 while (!kthread_should_stop())
470 init_test_configurtion(void)
473 * A maximum number of workers is defined as hard-coded
474 * value and set to USHRT_MAX. We add such gap just in
475 * case and for potential heavy stressing.
477 nr_threads
= clamp(nr_threads
, 1, (int) USHRT_MAX
);
479 /* Allocate the space for test instances. */
480 tdriver
= kvcalloc(nr_threads
, sizeof(*tdriver
), GFP_KERNEL
);
484 if (test_repeat_count
<= 0)
485 test_repeat_count
= 1;
487 if (test_loop_count
<= 0)
493 static void do_concurrent_test(void)
498 * Set some basic configurations plus sanity check.
500 ret
= init_test_configurtion();
505 * Put on hold all workers.
507 down_write(&prepare_for_test_rwsem
);
509 for (i
= 0; i
< nr_threads
; i
++) {
510 struct test_driver
*t
= &tdriver
[i
];
512 t
->task
= kthread_run(test_func
, t
, "vmalloc_test/%d", i
);
514 if (!IS_ERR(t
->task
))
516 atomic_inc(&test_n_undone
);
518 pr_err("Failed to start %d kthread\n", i
);
522 * Now let the workers do their job.
524 up_write(&prepare_for_test_rwsem
);
527 * Sleep quiet until all workers are done with 1 second
528 * interval. Since the test can take a lot of time we
529 * can run into a stack trace of the hung task. That is
530 * why we go with completion_timeout and HZ value.
533 ret
= wait_for_completion_timeout(&test_all_done_comp
, HZ
);
536 for (i
= 0; i
< nr_threads
; i
++) {
537 struct test_driver
*t
= &tdriver
[i
];
540 if (!IS_ERR(t
->task
))
541 kthread_stop(t
->task
);
543 for (j
= 0; j
< ARRAY_SIZE(test_case_array
); j
++) {
544 if (!((run_test_mask
& (1 << j
)) >> j
))
548 "Summary: %s passed: %d failed: %d repeat: %d loops: %d avg: %llu usec\n",
549 test_case_array
[j
].test_name
,
550 t
->data
[j
].test_passed
,
551 t
->data
[j
].test_failed
,
552 test_repeat_count
, test_loop_count
,
556 pr_info("All test took worker%d=%lu cycles\n",
557 i
, t
->stop
- t
->start
);
563 static int vmalloc_test_init(void)
565 do_concurrent_test();
566 return -EAGAIN
; /* Fail will directly unload the module */
569 static void vmalloc_test_exit(void)
573 module_init(vmalloc_test_init
)
574 module_exit(vmalloc_test_exit
)
576 MODULE_LICENSE("GPL");
577 MODULE_AUTHOR("Uladzislau Rezki");
578 MODULE_DESCRIPTION("vmalloc test module");