]>
Commit | Line | Data |
---|---|---|
4db40c94 | 1 | /* |
8020d79b | 2 | * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. |
4db40c94 | 3 | * |
909f1a2e | 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use |
4db40c94 RL |
5 | * this file except in compliance with the License. You can obtain a copy |
6 | * in the file LICENSE in the source distribution or at | |
7 | * https://www.openssl.org/source/license.html | |
8 | */ | |
9 | ||
10 | #include "../testutil.h" | |
579d0fab | 11 | #include "output.h" |
c5657cb7 | 12 | #include "tu_local.h" |
4db40c94 RL |
13 | |
14 | #include <string.h> | |
15 | #include <assert.h> | |
16 | ||
176db6dc | 17 | #include "internal/nelem.h" |
4db40c94 RL |
18 | #include <openssl/bio.h> |
19 | ||
ce506d27 RL |
20 | #include "platform.h" /* From libapps */ |
21 | ||
3a63c0ed AP |
22 | #ifdef _WIN32 |
23 | # define strdup _strdup | |
24 | #endif | |
25 | ||
a43ce58f | 26 | |
4db40c94 RL |
27 | /* |
28 | * Declares the structures needed to register each test case function. | |
29 | */ | |
30 | typedef struct test_info { | |
31 | const char *test_case_name; | |
91860165 | 32 | int (*test_fn) (void); |
4db40c94 RL |
33 | int (*param_test_fn)(int idx); |
34 | int num; | |
208d721a RL |
35 | |
36 | /* flags */ | |
37 | int subtest:1; | |
4db40c94 RL |
38 | } TEST_INFO; |
39 | ||
40 | static TEST_INFO all_tests[1024]; | |
41 | static int num_tests = 0; | |
a43ce58f SL |
42 | static int show_list = 0; |
43 | static int single_test = -1; | |
44 | static int single_iter = -1; | |
45 | static int level = 0; | |
5584fd7f | 46 | static int seed = 0; |
b3ab537b P |
47 | static int rand_order = 0; |
48 | ||
4db40c94 | 49 | /* |
a43ce58f | 50 | * A parameterised test runs a loop of test cases. |
4db40c94 RL |
51 | * |num_test_cases| counts the total number of test cases |
52 | * across all tests. | |
53 | */ | |
54 | static int num_test_cases = 0; | |
55 | ||
a43ce58f SL |
56 | static int process_shared_options(void); |
57 | ||
58 | ||
180c3fc7 | 59 | void add_test(const char *test_case_name, int (*test_fn) (void)) |
4db40c94 RL |
60 | { |
61 | assert(num_tests != OSSL_NELEM(all_tests)); | |
62 | all_tests[num_tests].test_case_name = test_case_name; | |
63 | all_tests[num_tests].test_fn = test_fn; | |
64 | all_tests[num_tests].num = -1; | |
65 | ++num_tests; | |
66 | ++num_test_cases; | |
67 | } | |
68 | ||
69 | void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), | |
208d721a | 70 | int num, int subtest) |
4db40c94 RL |
71 | { |
72 | assert(num_tests != OSSL_NELEM(all_tests)); | |
73 | all_tests[num_tests].test_case_name = test_case_name; | |
74 | all_tests[num_tests].param_test_fn = test_fn; | |
75 | all_tests[num_tests].num = num; | |
208d721a | 76 | all_tests[num_tests].subtest = subtest; |
4db40c94 RL |
77 | ++num_tests; |
78 | num_test_cases += num; | |
79 | } | |
80 | ||
5584fd7f P |
81 | static int gcd(int a, int b) |
82 | { | |
83 | while (b != 0) { | |
84 | int t = b; | |
85 | b = a % b; | |
86 | a = t; | |
87 | } | |
88 | return a; | |
89 | } | |
90 | ||
a43ce58f | 91 | static void set_seed(int s) |
4db40c94 | 92 | { |
a43ce58f SL |
93 | seed = s; |
94 | if (seed <= 0) | |
95 | seed = (int)time(NULL); | |
e9a5932d | 96 | test_random_seed(seed); |
a43ce58f | 97 | } |
208d721a | 98 | |
208d721a | 99 | |
a43ce58f SL |
100 | int setup_test_framework(int argc, char *argv[]) |
101 | { | |
102 | char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER"); | |
103 | char *TAP_levels = getenv("HARNESS_OSSL_LEVEL"); | |
104 | ||
105 | if (TAP_levels != NULL) | |
106 | level = 4 * atoi(TAP_levels); | |
66d8bd4b | 107 | test_adjust_streams_tap_level(level); |
b3ab537b P |
108 | if (test_seed != NULL) { |
109 | rand_order = 1; | |
a43ce58f | 110 | set_seed(atoi(test_seed)); |
b3ab537b P |
111 | } else { |
112 | set_seed(0); | |
113 | } | |
5584fd7f | 114 | |
ce506d27 RL |
115 | #if defined(OPENSSL_SYS_VMS) && defined(__DECC) |
116 | argv = copy_argv(&argc, argv); | |
117 | #elif defined(_WIN32) | |
118 | /* | |
119 | * Replace argv[] with UTF-8 encoded strings. | |
120 | */ | |
121 | win32_utf8argv(&argc, &argv); | |
122 | #endif | |
123 | ||
a43ce58f SL |
124 | if (!opt_init(argc, argv, test_get_options())) |
125 | return 0; | |
126 | return 1; | |
4db40c94 RL |
127 | } |
128 | ||
a43ce58f SL |
129 | |
130 | /* | |
131 | * This can only be called after setup() has run, since num_tests and | |
132 | * all_tests[] are setup at this point | |
133 | */ | |
134 | static int check_single_test_params(char *name, char *testname, char *itname) | |
135 | { | |
136 | if (name != NULL) { | |
137 | int i; | |
138 | for (i = 0; i < num_tests; ++i) { | |
139 | if (strcmp(name, all_tests[i].test_case_name) == 0) { | |
140 | single_test = 1 + i; | |
141 | break; | |
4bd8b240 | 142 | } |
a43ce58f | 143 | } |
4bd8b240 | 144 | if (i >= num_tests) |
dd6b2706 | 145 | single_test = atoi(name); |
a43ce58f SL |
146 | } |
147 | ||
148 | ||
149 | /* if only iteration is specified, assume we want the first test */ | |
150 | if (single_test == -1 && single_iter != -1) | |
151 | single_test = 1; | |
152 | ||
153 | if (single_test != -1) { | |
154 | if (single_test < 1 || single_test > num_tests) { | |
155 | test_printf_stderr("Invalid -%s value " | |
156 | "(Value must be a valid test name OR a value between %d..%d)\n", | |
157 | testname, 1, num_tests); | |
158 | return 0; | |
159 | } | |
160 | } | |
161 | if (single_iter != -1) { | |
162 | if (all_tests[single_test - 1].num == -1) { | |
163 | test_printf_stderr("-%s option is not valid for test %d:%s\n", | |
164 | itname, | |
165 | single_test, | |
166 | all_tests[single_test - 1].test_case_name); | |
167 | return 0; | |
168 | } else if (single_iter < 1 | |
169 | || single_iter > all_tests[single_test - 1].num) { | |
170 | test_printf_stderr("Invalid -%s value for test %d:%s\t" | |
171 | "(Value must be in the range %d..%d)\n", | |
172 | itname, single_test, | |
173 | all_tests[single_test - 1].test_case_name, | |
174 | 1, all_tests[single_test - 1].num); | |
175 | return 0; | |
176 | } | |
177 | } | |
178 | return 1; | |
179 | } | |
180 | ||
181 | static int process_shared_options(void) | |
182 | { | |
183 | OPTION_CHOICE_DEFAULT o; | |
184 | int value; | |
185 | int ret = -1; | |
186 | char *flag_test = ""; | |
187 | char *flag_iter = ""; | |
188 | char *testname = NULL; | |
189 | ||
190 | opt_begin(); | |
191 | while ((o = opt_next()) != OPT_EOF) { | |
192 | switch (o) { | |
193 | /* Ignore any test options at this level */ | |
194 | default: | |
195 | break; | |
196 | case OPT_ERR: | |
197 | return ret; | |
198 | case OPT_TEST_HELP: | |
199 | opt_help(test_get_options()); | |
200 | return 0; | |
201 | case OPT_TEST_LIST: | |
202 | show_list = 1; | |
203 | break; | |
204 | case OPT_TEST_SINGLE: | |
205 | flag_test = opt_flag(); | |
206 | testname = opt_arg(); | |
207 | break; | |
208 | case OPT_TEST_ITERATION: | |
209 | flag_iter = opt_flag(); | |
210 | if (!opt_int(opt_arg(), &single_iter)) | |
211 | goto end; | |
212 | break; | |
213 | case OPT_TEST_INDENT: | |
214 | if (!opt_int(opt_arg(), &value)) | |
215 | goto end; | |
216 | level = 4 * value; | |
66d8bd4b | 217 | test_adjust_streams_tap_level(level); |
a43ce58f SL |
218 | break; |
219 | case OPT_TEST_SEED: | |
220 | if (!opt_int(opt_arg(), &value)) | |
221 | goto end; | |
222 | set_seed(value); | |
223 | break; | |
224 | } | |
225 | } | |
226 | if (!check_single_test_params(testname, flag_test, flag_iter)) | |
227 | goto end; | |
228 | ret = 1; | |
229 | end: | |
230 | return ret; | |
231 | } | |
232 | ||
233 | ||
ad887416 | 234 | int pulldown_test_framework(int ret) |
4db40c94 | 235 | { |
180c3fc7 | 236 | set_test_title(NULL); |
4db40c94 RL |
237 | return ret; |
238 | } | |
239 | ||
240 | static void finalize(int success) | |
241 | { | |
242 | if (success) | |
243 | ERR_clear_error(); | |
244 | else | |
68e49bf2 | 245 | ERR_print_errors_cb(openssl_error_cb, NULL); |
4db40c94 RL |
246 | } |
247 | ||
1d0f116e | 248 | static char *test_title = NULL; |
b73c5e05 RL |
249 | |
250 | void set_test_title(const char *title) | |
251 | { | |
1d0f116e RS |
252 | free(test_title); |
253 | test_title = title == NULL ? NULL : strdup(title); | |
b73c5e05 RL |
254 | } |
255 | ||
c5f7a996 P |
256 | PRINTF_FORMAT(2, 3) static void test_verdict(int verdict, |
257 | const char *description, ...) | |
ad887416 P |
258 | { |
259 | va_list ap; | |
260 | ||
261 | test_flush_stdout(); | |
262 | test_flush_stderr(); | |
263 | ||
f7f10de3 | 264 | if (verdict == 0 && seed != 0) |
a21a1c23 | 265 | test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed); |
66d8bd4b | 266 | test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok"); |
c5f7a996 | 267 | va_start(ap, description); |
66d8bd4b | 268 | test_vprintf_tapout(description, ap); |
c5f7a996 P |
269 | va_end(ap); |
270 | if (verdict == TEST_SKIP_CODE) | |
66d8bd4b RL |
271 | test_printf_tapout(" # skipped"); |
272 | test_printf_tapout("\n"); | |
f7f10de3 | 273 | test_flush_tapout(); |
ad887416 P |
274 | } |
275 | ||
4db40c94 RL |
276 | int run_tests(const char *test_prog_name) |
277 | { | |
278 | int num_failed = 0; | |
ad887416 | 279 | int verdict = 1; |
5584fd7f P |
280 | int ii, i, jj, j, jstep; |
281 | int permute[OSSL_NELEM(all_tests)]; | |
4db40c94 | 282 | |
a43ce58f SL |
283 | i = process_shared_options(); |
284 | if (i == 0) | |
285 | return EXIT_SUCCESS; | |
286 | if (i == -1) | |
287 | return EXIT_FAILURE; | |
288 | ||
0b10da80 | 289 | if (num_tests < 1) { |
66d8bd4b | 290 | test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name); |
a43ce58f | 291 | } else if (show_list == 0 && single_test == -1) { |
f7f10de3 | 292 | if (level > 0) { |
66d8bd4b | 293 | test_printf_stdout("Subtest: %s\n", test_prog_name); |
f7f10de3 P |
294 | test_flush_stdout(); |
295 | } | |
66d8bd4b | 296 | test_printf_tapout("1..%d\n", num_tests); |
0b10da80 | 297 | } |
a43ce58f | 298 | |
f7f10de3 | 299 | test_flush_tapout(); |
4db40c94 | 300 | |
5584fd7f P |
301 | for (i = 0; i < num_tests; i++) |
302 | permute[i] = i; | |
b3ab537b | 303 | if (rand_order != 0) |
5584fd7f | 304 | for (i = num_tests - 1; i >= 1; i--) { |
e9a5932d | 305 | j = test_random() % (1 + i); |
5584fd7f P |
306 | ii = permute[j]; |
307 | permute[j] = permute[i]; | |
308 | permute[i] = ii; | |
309 | } | |
310 | ||
311 | for (ii = 0; ii != num_tests; ++ii) { | |
312 | i = permute[ii]; | |
a43ce58f SL |
313 | |
314 | if (single_test != -1 && ((i+1) != single_test)) { | |
315 | continue; | |
316 | } | |
317 | else if (show_list) { | |
318 | if (all_tests[i].num != -1) { | |
66d8bd4b | 319 | test_printf_tapout("%d - %s (%d..%d)\n", ii + 1, |
a43ce58f SL |
320 | all_tests[i].test_case_name, 1, |
321 | all_tests[i].num); | |
322 | } else { | |
66d8bd4b | 323 | test_printf_tapout("%d - %s\n", ii + 1, |
a43ce58f SL |
324 | all_tests[i].test_case_name); |
325 | } | |
f7f10de3 | 326 | test_flush_tapout(); |
a43ce58f | 327 | } else if (all_tests[i].num == -1) { |
b73c5e05 | 328 | set_test_title(all_tests[i].test_case_name); |
c5f7a996 | 329 | verdict = all_tests[i].test_fn(); |
c5f7a996 | 330 | finalize(verdict != 0); |
4519ea90 | 331 | test_verdict(verdict, "%d - %s", ii + 1, test_title); |
cb30e46b MC |
332 | if (verdict == 0) |
333 | num_failed++; | |
4db40c94 | 334 | } else { |
208d721a RL |
335 | int num_failed_inner = 0; |
336 | ||
c5f7a996 | 337 | verdict = TEST_SKIP_CODE; |
208d721a | 338 | level += 4; |
66d8bd4b | 339 | test_adjust_streams_tap_level(level); |
a43ce58f | 340 | if (all_tests[i].subtest && single_iter == -1) { |
66d8bd4b | 341 | test_printf_stdout("Subtest: %s\n", |
603ddbdb | 342 | all_tests[i].test_case_name); |
66d8bd4b | 343 | test_printf_tapout("%d..%d\n", 1, all_tests[i].num); |
208d721a | 344 | test_flush_stdout(); |
f7f10de3 | 345 | test_flush_tapout(); |
208d721a RL |
346 | } |
347 | ||
5584fd7f | 348 | j = -1; |
b3ab537b | 349 | if (rand_order == 0 || all_tests[i].num < 3) |
5584fd7f P |
350 | jstep = 1; |
351 | else | |
352 | do | |
e9a5932d | 353 | jstep = test_random() % all_tests[i].num; |
5584fd7f P |
354 | while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1); |
355 | ||
356 | for (jj = 0; jj < all_tests[i].num; jj++) { | |
c5f7a996 | 357 | int v; |
b73c5e05 | 358 | |
5584fd7f | 359 | j = (j + jstep) % all_tests[i].num; |
a43ce58f SL |
360 | if (single_iter != -1 && ((jj + 1) != single_iter)) |
361 | continue; | |
b73c5e05 | 362 | set_test_title(NULL); |
c5f7a996 | 363 | v = all_tests[i].param_test_fn(j); |
4db40c94 | 364 | |
c5f7a996 | 365 | if (v == 0) { |
208d721a | 366 | ++num_failed_inner; |
c5f7a996 P |
367 | verdict = 0; |
368 | } else if (v != TEST_SKIP_CODE && verdict != 0) { | |
369 | verdict = 1; | |
370 | } | |
208d721a | 371 | |
c5f7a996 | 372 | finalize(v != 0); |
208d721a RL |
373 | |
374 | if (all_tests[i].subtest) { | |
b73c5e05 | 375 | if (test_title != NULL) |
c5f7a996 | 376 | test_verdict(v, "%d - %s", jj + 1, test_title); |
b73c5e05 | 377 | else |
c5f7a996 | 378 | test_verdict(v, "%d - iteration %d", jj + 1, j + 1); |
4db40c94 | 379 | } |
4db40c94 | 380 | } |
208d721a RL |
381 | |
382 | level -= 4; | |
66d8bd4b | 383 | test_adjust_streams_tap_level(level); |
c5f7a996 | 384 | if (verdict == 0) |
208d721a | 385 | ++num_failed; |
ad887416 P |
386 | test_verdict(verdict, "%d - %s", ii + 1, |
387 | all_tests[i].test_case_name); | |
4db40c94 RL |
388 | } |
389 | } | |
208d721a | 390 | if (num_failed != 0) |
4db40c94 | 391 | return EXIT_FAILURE; |
4db40c94 RL |
392 | return EXIT_SUCCESS; |
393 | } | |
394 | ||
4483fbae F |
395 | /* |
396 | * Glue an array of strings together and return it as an allocated string. | |
397 | * Optionally return the whole length of this string in |out_len| | |
398 | */ | |
399 | char *glue_strings(const char *list[], size_t *out_len) | |
400 | { | |
401 | size_t len = 0; | |
402 | char *p, *ret; | |
403 | int i; | |
404 | ||
405 | for (i = 0; list[i] != NULL; i++) | |
406 | len += strlen(list[i]); | |
407 | ||
408 | if (out_len != NULL) | |
409 | *out_len = len; | |
410 | ||
411 | if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1))) | |
412 | return NULL; | |
413 | ||
414 | for (i = 0; list[i] != NULL; i++) | |
415 | p += strlen(strcpy(p, list[i])); | |
416 | ||
417 | return ret; | |
418 | } | |
419 | ||
1a2a3a42 MC |
420 | char *test_mk_file_path(const char *dir, const char *file) |
421 | { | |
422 | # ifndef OPENSSL_SYS_VMS | |
423 | const char *sep = "/"; | |
424 | # else | |
425 | const char *sep = ""; | |
426 | # endif | |
427 | size_t len = strlen(dir) + strlen(sep) + strlen(file) + 1; | |
428 | char *full_file = OPENSSL_zalloc(len); | |
429 | ||
430 | if (full_file != NULL) { | |
431 | OPENSSL_strlcpy(full_file, dir, len); | |
432 | OPENSSL_strlcat(full_file, sep, len); | |
433 | OPENSSL_strlcat(full_file, file, len); | |
434 | } | |
435 | ||
436 | return full_file; | |
437 | } |