]>
Commit | Line | Data |
---|---|---|
4db40c94 | 1 | /* |
1212818e | 2 | * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved. |
4db40c94 | 3 | * |
909f1a2e | 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use |
4db40c94 RL |
5 | * this file except in compliance with the License. You can obtain a copy |
6 | * in the file LICENSE in the source distribution or at | |
7 | * https://www.openssl.org/source/license.html | |
8 | */ | |
9 | ||
10 | #include "../testutil.h" | |
579d0fab | 11 | #include "output.h" |
c5657cb7 | 12 | #include "tu_local.h" |
4db40c94 RL |
13 | |
14 | #include <string.h> | |
15 | #include <assert.h> | |
16 | ||
176db6dc | 17 | #include "internal/nelem.h" |
4db40c94 RL |
18 | #include <openssl/bio.h> |
19 | ||
3a63c0ed AP |
20 | #ifdef _WIN32 |
21 | # define strdup _strdup | |
22 | #endif | |
23 | ||
a43ce58f | 24 | |
4db40c94 RL |
25 | /* |
26 | * Declares the structures needed to register each test case function. | |
27 | */ | |
28 | typedef struct test_info { | |
29 | const char *test_case_name; | |
91860165 | 30 | int (*test_fn) (void); |
4db40c94 RL |
31 | int (*param_test_fn)(int idx); |
32 | int num; | |
208d721a RL |
33 | |
34 | /* flags */ | |
35 | int subtest:1; | |
4db40c94 RL |
36 | } TEST_INFO; |
37 | ||
38 | static TEST_INFO all_tests[1024]; | |
39 | static int num_tests = 0; | |
a43ce58f SL |
40 | static int show_list = 0; |
41 | static int single_test = -1; | |
42 | static int single_iter = -1; | |
43 | static int level = 0; | |
5584fd7f | 44 | static int seed = 0; |
4db40c94 | 45 | /* |
a43ce58f | 46 | * A parameterised test runs a loop of test cases. |
4db40c94 RL |
47 | * |num_test_cases| counts the total number of test cases |
48 | * across all tests. | |
49 | */ | |
50 | static int num_test_cases = 0; | |
51 | ||
a43ce58f SL |
52 | static int process_shared_options(void); |
53 | ||
54 | ||
180c3fc7 | 55 | void add_test(const char *test_case_name, int (*test_fn) (void)) |
4db40c94 RL |
56 | { |
57 | assert(num_tests != OSSL_NELEM(all_tests)); | |
58 | all_tests[num_tests].test_case_name = test_case_name; | |
59 | all_tests[num_tests].test_fn = test_fn; | |
60 | all_tests[num_tests].num = -1; | |
61 | ++num_tests; | |
62 | ++num_test_cases; | |
63 | } | |
64 | ||
65 | void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), | |
208d721a | 66 | int num, int subtest) |
4db40c94 RL |
67 | { |
68 | assert(num_tests != OSSL_NELEM(all_tests)); | |
69 | all_tests[num_tests].test_case_name = test_case_name; | |
70 | all_tests[num_tests].param_test_fn = test_fn; | |
71 | all_tests[num_tests].num = num; | |
208d721a | 72 | all_tests[num_tests].subtest = subtest; |
4db40c94 RL |
73 | ++num_tests; |
74 | num_test_cases += num; | |
75 | } | |
76 | ||
208d721a RL |
77 | int subtest_level(void) |
78 | { | |
79 | return level; | |
80 | } | |
81 | ||
4db40c94 | 82 | #ifndef OPENSSL_NO_CRYPTO_MDEBUG |
12bd4e14 | 83 | static int should_report_leaks(void) |
4db40c94 RL |
84 | { |
85 | /* | |
86 | * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0 | |
87 | * can be used to disable leak checking at runtime. | |
88 | * Note this only works when running the test binary manually; | |
89 | * the test harness always enables OPENSSL_DEBUG_MEMORY. | |
90 | */ | |
91 | char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY"); | |
92 | ||
93 | return mem_debug_env == NULL | |
94 | || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, "")); | |
95 | } | |
96 | #endif | |
97 | ||
5584fd7f P |
98 | static int gcd(int a, int b) |
99 | { | |
100 | while (b != 0) { | |
101 | int t = b; | |
102 | b = a % b; | |
103 | a = t; | |
104 | } | |
105 | return a; | |
106 | } | |
107 | ||
a43ce58f | 108 | static void set_seed(int s) |
4db40c94 | 109 | { |
a43ce58f SL |
110 | seed = s; |
111 | if (seed <= 0) | |
112 | seed = (int)time(NULL); | |
113 | test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed); | |
114 | test_flush_stdout(); | |
115 | srand(seed); | |
116 | } | |
208d721a | 117 | |
208d721a | 118 | |
a43ce58f SL |
119 | int setup_test_framework(int argc, char *argv[]) |
120 | { | |
121 | char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER"); | |
122 | char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); | |
123 | ||
124 | if (TAP_levels != NULL) | |
125 | level = 4 * atoi(TAP_levels); | |
126 | if (test_seed != NULL) | |
127 | set_seed(atoi(test_seed)); | |
5584fd7f | 128 | |
4db40c94 RL |
129 | #ifndef OPENSSL_NO_CRYPTO_MDEBUG |
130 | if (should_report_leaks()) { | |
131 | CRYPTO_set_mem_debug(1); | |
132 | CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON); | |
133 | } | |
134 | #endif | |
a43ce58f SL |
135 | if (!opt_init(argc, argv, test_get_options())) |
136 | return 0; | |
137 | return 1; | |
4db40c94 RL |
138 | } |
139 | ||
a43ce58f SL |
140 | |
141 | /* | |
142 | * This can only be called after setup() has run, since num_tests and | |
143 | * all_tests[] are setup at this point | |
144 | */ | |
145 | static int check_single_test_params(char *name, char *testname, char *itname) | |
146 | { | |
147 | if (name != NULL) { | |
148 | int i; | |
149 | for (i = 0; i < num_tests; ++i) { | |
150 | if (strcmp(name, all_tests[i].test_case_name) == 0) { | |
151 | single_test = 1 + i; | |
152 | break; | |
153 | } | |
154 | } | |
155 | if (i >= num_tests) | |
156 | single_test = atoi(name); | |
157 | } | |
158 | ||
159 | ||
160 | /* if only iteration is specified, assume we want the first test */ | |
161 | if (single_test == -1 && single_iter != -1) | |
162 | single_test = 1; | |
163 | ||
164 | if (single_test != -1) { | |
165 | if (single_test < 1 || single_test > num_tests) { | |
166 | test_printf_stderr("Invalid -%s value " | |
167 | "(Value must be a valid test name OR a value between %d..%d)\n", | |
168 | testname, 1, num_tests); | |
169 | return 0; | |
170 | } | |
171 | } | |
172 | if (single_iter != -1) { | |
173 | if (all_tests[single_test - 1].num == -1) { | |
174 | test_printf_stderr("-%s option is not valid for test %d:%s\n", | |
175 | itname, | |
176 | single_test, | |
177 | all_tests[single_test - 1].test_case_name); | |
178 | return 0; | |
179 | } else if (single_iter < 1 | |
180 | || single_iter > all_tests[single_test - 1].num) { | |
181 | test_printf_stderr("Invalid -%s value for test %d:%s\t" | |
182 | "(Value must be in the range %d..%d)\n", | |
183 | itname, single_test, | |
184 | all_tests[single_test - 1].test_case_name, | |
185 | 1, all_tests[single_test - 1].num); | |
186 | return 0; | |
187 | } | |
188 | } | |
189 | return 1; | |
190 | } | |
191 | ||
192 | static int process_shared_options(void) | |
193 | { | |
194 | OPTION_CHOICE_DEFAULT o; | |
195 | int value; | |
196 | int ret = -1; | |
197 | char *flag_test = ""; | |
198 | char *flag_iter = ""; | |
199 | char *testname = NULL; | |
200 | ||
201 | opt_begin(); | |
202 | while ((o = opt_next()) != OPT_EOF) { | |
203 | switch (o) { | |
204 | /* Ignore any test options at this level */ | |
205 | default: | |
206 | break; | |
207 | case OPT_ERR: | |
208 | return ret; | |
209 | case OPT_TEST_HELP: | |
210 | opt_help(test_get_options()); | |
211 | return 0; | |
212 | case OPT_TEST_LIST: | |
213 | show_list = 1; | |
214 | break; | |
215 | case OPT_TEST_SINGLE: | |
216 | flag_test = opt_flag(); | |
217 | testname = opt_arg(); | |
218 | break; | |
219 | case OPT_TEST_ITERATION: | |
220 | flag_iter = opt_flag(); | |
221 | if (!opt_int(opt_arg(), &single_iter)) | |
222 | goto end; | |
223 | break; | |
224 | case OPT_TEST_INDENT: | |
225 | if (!opt_int(opt_arg(), &value)) | |
226 | goto end; | |
227 | level = 4 * value; | |
228 | break; | |
229 | case OPT_TEST_SEED: | |
230 | if (!opt_int(opt_arg(), &value)) | |
231 | goto end; | |
232 | set_seed(value); | |
233 | break; | |
234 | } | |
235 | } | |
236 | if (!check_single_test_params(testname, flag_test, flag_iter)) | |
237 | goto end; | |
238 | ret = 1; | |
239 | end: | |
240 | return ret; | |
241 | } | |
242 | ||
243 | ||
ad887416 | 244 | int pulldown_test_framework(int ret) |
4db40c94 | 245 | { |
180c3fc7 | 246 | set_test_title(NULL); |
4db40c94 | 247 | #ifndef OPENSSL_NO_CRYPTO_MDEBUG |
68e49bf2 RL |
248 | if (should_report_leaks() |
249 | && CRYPTO_mem_leaks_cb(openssl_error_cb, NULL) <= 0) | |
4db40c94 RL |
250 | return EXIT_FAILURE; |
251 | #endif | |
4db40c94 RL |
252 | return ret; |
253 | } | |
254 | ||
255 | static void finalize(int success) | |
256 | { | |
257 | if (success) | |
258 | ERR_clear_error(); | |
259 | else | |
68e49bf2 | 260 | ERR_print_errors_cb(openssl_error_cb, NULL); |
4db40c94 RL |
261 | } |
262 | ||
1d0f116e | 263 | static char *test_title = NULL; |
b73c5e05 RL |
264 | |
265 | void set_test_title(const char *title) | |
266 | { | |
1d0f116e RS |
267 | free(test_title); |
268 | test_title = title == NULL ? NULL : strdup(title); | |
b73c5e05 RL |
269 | } |
270 | ||
ad887416 P |
271 | PRINTF_FORMAT(2, 3) static void test_verdict(int pass, const char *extra, ...) |
272 | { | |
273 | va_list ap; | |
274 | ||
275 | test_flush_stdout(); | |
276 | test_flush_stderr(); | |
277 | ||
278 | test_printf_stdout("%*s%s", level, "", pass ? "ok" : "not ok"); | |
279 | if (extra != NULL) { | |
280 | test_printf_stdout(" "); | |
281 | va_start(ap, extra); | |
282 | test_vprintf_stdout(extra, ap); | |
283 | va_end(ap); | |
284 | } | |
285 | test_printf_stdout("\n"); | |
286 | test_flush_stdout(); | |
287 | } | |
288 | ||
4db40c94 RL |
289 | int run_tests(const char *test_prog_name) |
290 | { | |
291 | int num_failed = 0; | |
ad887416 | 292 | int verdict = 1; |
5584fd7f P |
293 | int ii, i, jj, j, jstep; |
294 | int permute[OSSL_NELEM(all_tests)]; | |
4db40c94 | 295 | |
a43ce58f SL |
296 | i = process_shared_options(); |
297 | if (i == 0) | |
298 | return EXIT_SUCCESS; | |
299 | if (i == -1) | |
300 | return EXIT_FAILURE; | |
301 | ||
0b10da80 | 302 | if (num_tests < 1) { |
603ddbdb RL |
303 | test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "", |
304 | test_prog_name); | |
a43ce58f | 305 | } else if (show_list == 0 && single_test == -1) { |
0b10da80 RL |
306 | if (level > 0) |
307 | test_printf_stdout("%*s# Subtest: %s\n", level, "", test_prog_name); | |
603ddbdb | 308 | test_printf_stdout("%*s1..%d\n", level, "", num_tests); |
0b10da80 | 309 | } |
a43ce58f | 310 | |
4db40c94 RL |
311 | test_flush_stdout(); |
312 | ||
5584fd7f P |
313 | for (i = 0; i < num_tests; i++) |
314 | permute[i] = i; | |
315 | if (seed != 0) | |
316 | for (i = num_tests - 1; i >= 1; i--) { | |
317 | j = rand() % (1 + i); | |
318 | ii = permute[j]; | |
319 | permute[j] = permute[i]; | |
320 | permute[i] = ii; | |
321 | } | |
322 | ||
323 | for (ii = 0; ii != num_tests; ++ii) { | |
324 | i = permute[ii]; | |
a43ce58f SL |
325 | |
326 | if (single_test != -1 && ((i+1) != single_test)) { | |
327 | continue; | |
328 | } | |
329 | else if (show_list) { | |
330 | if (all_tests[i].num != -1) { | |
331 | test_printf_stdout("%d - %s (%d..%d)\n", ii + 1, | |
332 | all_tests[i].test_case_name, 1, | |
333 | all_tests[i].num); | |
334 | } else { | |
335 | test_printf_stdout("%d - %s\n", ii + 1, | |
336 | all_tests[i].test_case_name); | |
337 | } | |
338 | test_flush_stdout(); | |
339 | } else if (all_tests[i].num == -1) { | |
b73c5e05 RL |
340 | int ret = 0; |
341 | ||
342 | set_test_title(all_tests[i].test_case_name); | |
343 | ret = all_tests[i].test_fn(); | |
ad887416 | 344 | verdict = 1; |
4db40c94 | 345 | if (!ret) { |
ad887416 | 346 | verdict = 0; |
4db40c94 RL |
347 | ++num_failed; |
348 | } | |
ad887416 | 349 | test_verdict(verdict, "%d - %s", ii + 1, test_title); |
4db40c94 RL |
350 | finalize(ret); |
351 | } else { | |
208d721a RL |
352 | int num_failed_inner = 0; |
353 | ||
354 | level += 4; | |
a43ce58f | 355 | if (all_tests[i].subtest && single_iter == -1) { |
603ddbdb RL |
356 | test_printf_stdout("%*s# Subtest: %s\n", level, "", |
357 | all_tests[i].test_case_name); | |
358 | test_printf_stdout("%*s%d..%d\n", level, "", 1, | |
359 | all_tests[i].num); | |
208d721a RL |
360 | test_flush_stdout(); |
361 | } | |
362 | ||
5584fd7f P |
363 | j = -1; |
364 | if (seed == 0 || all_tests[i].num < 3) | |
365 | jstep = 1; | |
366 | else | |
367 | do | |
368 | jstep = rand() % all_tests[i].num; | |
369 | while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1); | |
370 | ||
371 | for (jj = 0; jj < all_tests[i].num; jj++) { | |
372 | int ret; | |
b73c5e05 | 373 | |
5584fd7f | 374 | j = (j + jstep) % all_tests[i].num; |
a43ce58f SL |
375 | if (single_iter != -1 && ((jj + 1) != single_iter)) |
376 | continue; | |
b73c5e05 RL |
377 | set_test_title(NULL); |
378 | ret = all_tests[i].param_test_fn(j); | |
4db40c94 | 379 | |
208d721a RL |
380 | if (!ret) |
381 | ++num_failed_inner; | |
382 | ||
383 | finalize(ret); | |
384 | ||
385 | if (all_tests[i].subtest) { | |
ad887416 | 386 | verdict = 1; |
208d721a | 387 | if (!ret) { |
ad887416 | 388 | verdict = 0; |
208d721a RL |
389 | ++num_failed_inner; |
390 | } | |
b73c5e05 | 391 | if (test_title != NULL) |
ad887416 | 392 | test_verdict(verdict, "%d - %s", jj + 1, test_title); |
b73c5e05 | 393 | else |
ad887416 P |
394 | test_verdict(verdict, "%d - iteration %d", |
395 | jj + 1, j + 1); | |
4db40c94 | 396 | } |
4db40c94 | 397 | } |
208d721a RL |
398 | |
399 | level -= 4; | |
ad887416 | 400 | verdict = 1; |
208d721a | 401 | if (num_failed_inner) { |
ad887416 | 402 | verdict = 0; |
208d721a RL |
403 | ++num_failed; |
404 | } | |
ad887416 P |
405 | test_verdict(verdict, "%d - %s", ii + 1, |
406 | all_tests[i].test_case_name); | |
4db40c94 RL |
407 | } |
408 | } | |
208d721a | 409 | if (num_failed != 0) |
4db40c94 | 410 | return EXIT_FAILURE; |
4db40c94 RL |
411 | return EXIT_SUCCESS; |
412 | } | |
413 | ||
4483fbae F |
414 | /* |
415 | * Glue an array of strings together and return it as an allocated string. | |
416 | * Optionally return the whole length of this string in |out_len| | |
417 | */ | |
418 | char *glue_strings(const char *list[], size_t *out_len) | |
419 | { | |
420 | size_t len = 0; | |
421 | char *p, *ret; | |
422 | int i; | |
423 | ||
424 | for (i = 0; list[i] != NULL; i++) | |
425 | len += strlen(list[i]); | |
426 | ||
427 | if (out_len != NULL) | |
428 | *out_len = len; | |
429 | ||
430 | if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1))) | |
431 | return NULL; | |
432 | ||
433 | for (i = 0; list[i] != NULL; i++) | |
434 | p += strlen(strcpy(p, list[i])); | |
435 | ||
436 | return ret; | |
437 | } | |
438 |