test/testutil/driver.c

   1 /*
   2  * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License 2.0 (the "License").  You may not use
   5  * this file except in compliance with the License.  You can obtain a copy
   6  * in the file LICENSE in the source distribution or at
   7  * https://www.openssl.org/source/license.html
   8  */
   9
  10 #include "../testutil.h"
  11 #include "output.h"
  12 #include "tu_local.h"
  13
  14 #include <string.h>
  15 #include <assert.h>
  16
  17 #include "internal/nelem.h"
  18 #include <openssl/bio.h>
  19
  20 #include "platform.h"            /* From libapps */
  21
  22 #ifdef _WIN32
  23 # define strdup _strdup
  24 #endif
  25
  26
  27 /*
  28  * Declares the structures needed to register each test case function.
  29  */
  30 typedef struct test_info {
  31     const char *test_case_name;
  32     int (*test_fn) (void);
  33     int (*param_test_fn)(int idx);
  34     int num;
  35
  36     /* flags */
  37     int subtest:1;
  38 } TEST_INFO;
  39
  40 static TEST_INFO all_tests[1024];
  41 static int num_tests = 0;
  42 static int show_list = 0;
  43 static int single_test = -1;
  44 static int single_iter = -1;
  45 static int level = 0;
  46 static int seed = 0;
  47 static int rand_order = 0;
  48
  49 /*
  50  * A parameterised test runs a loop of test cases.
  51  * |num_test_cases| counts the total number of non-subtest test cases
  52  * across all tests.
  53  */
  54 static int num_test_cases = 0;
  55
  56 static int process_shared_options(void);
  57
  58
  59 void add_test(const char *test_case_name, int (*test_fn) (void))
  60 {
  61     assert(num_tests != OSSL_NELEM(all_tests));
  62     all_tests[num_tests].test_case_name = test_case_name;
  63     all_tests[num_tests].test_fn = test_fn;
  64     all_tests[num_tests].num = -1;
  65     ++num_tests;
  66     ++num_test_cases;
  67 }
  68
  69 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
  70                    int num, int subtest)
  71 {
  72     assert(num_tests != OSSL_NELEM(all_tests));
  73     all_tests[num_tests].test_case_name = test_case_name;
  74     all_tests[num_tests].param_test_fn = test_fn;
  75     all_tests[num_tests].num = num;
  76     all_tests[num_tests].subtest = subtest;
  77     ++num_tests;
  78     if (subtest)
  79         ++num_test_cases;
  80     else
  81         num_test_cases += num;
  82 }
  83
  84 static int gcd(int a, int b)
  85 {
  86     while (b != 0) {
  87         int t = b;
  88         b = a % b;
  89         a = t;
  90     }
  91     return a;
  92 }
  93
  94 static void set_seed(int s)
  95 {
  96     seed = s;
  97     if (seed <= 0)
  98         seed = (int)time(NULL);
  99     test_random_seed(seed);
 100 }
 101
 102
 103 int setup_test_framework(int argc, char *argv[])
 104 {
 105     char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
 106     char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
 107
 108     if (TAP_levels != NULL)
 109         level = 4 * atoi(TAP_levels);
 110     test_adjust_streams_tap_level(level);
 111     if (test_seed != NULL) {
 112         rand_order = 1;
 113         set_seed(atoi(test_seed));
 114     } else {
 115         set_seed(0);
 116     }
 117
 118 #if defined(OPENSSL_SYS_VMS) && defined(__DECC)
 119     argv = copy_argv(&argc, argv);
 120 #elif defined(_WIN32)
 121     /*
 122      * Replace argv[] with UTF-8 encoded strings.
 123      */
 124     win32_utf8argv(&argc, &argv);
 125 #endif
 126
 127     if (!opt_init(argc, argv, test_get_options()))
 128         return 0;
 129     return 1;
 130 }
 131
 132
 133 /*
 134  * This can only be called after setup() has run, since num_tests and
 135  * all_tests[] are setup at this point
 136  */
 137 static int check_single_test_params(char *name, char *testname, char *itname)
 138 {
 139     if (name != NULL) {
 140         int i;
 141         for (i = 0; i < num_tests; ++i) {
 142             if (strcmp(name, all_tests[i].test_case_name) == 0) {
 143                 single_test = 1 + i;
 144                 break;
 145             }
 146         }
 147         if (i >= num_tests)
 148             single_test = atoi(name);
 149     }
 150
 151
 152     /* if only iteration is specified, assume we want the first test */
 153     if (single_test == -1 && single_iter != -1)
 154         single_test = 1;
 155
 156     if (single_test != -1) {
 157         if (single_test < 1 || single_test > num_tests) {
 158             test_printf_stderr("Invalid -%s value "
 159                                "(Value must be a valid test name OR a value between %d..%d)\n",
 160                                testname, 1, num_tests);
 161             return 0;
 162         }
 163     }
 164     if (single_iter != -1) {
 165         if (all_tests[single_test - 1].num == -1) {
 166             test_printf_stderr("-%s option is not valid for test %d:%s\n",
 167                                itname,
 168                                single_test,
 169                                all_tests[single_test - 1].test_case_name);
 170             return 0;
 171         } else if (single_iter < 1
 172                    || single_iter > all_tests[single_test - 1].num) {
 173             test_printf_stderr("Invalid -%s value for test %d:%s\t"
 174                                "(Value must be in the range %d..%d)\n",
 175                                itname, single_test,
 176                                all_tests[single_test - 1].test_case_name,
 177                                1, all_tests[single_test - 1].num);
 178             return 0;
 179         }
 180     }
 181     return 1;
 182 }
 183
 184 static int process_shared_options(void)
 185 {
 186     OPTION_CHOICE_DEFAULT o;
 187     int value;
 188     int ret = -1;
 189     char *flag_test = "";
 190     char *flag_iter = "";
 191     char *testname = NULL;
 192
 193     opt_begin();
 194     while ((o = opt_next()) != OPT_EOF) {
 195         switch (o) {
 196         /* Ignore any test options at this level */
 197         default:
 198             break;
 199         case OPT_ERR:
 200             return ret;
 201         case OPT_TEST_HELP:
 202             opt_help(test_get_options());
 203             return 0;
 204         case OPT_TEST_LIST:
 205             show_list = 1;
 206             break;
 207         case OPT_TEST_SINGLE:
 208             flag_test = opt_flag();
 209             testname = opt_arg();
 210             break;
 211         case OPT_TEST_ITERATION:
 212             flag_iter = opt_flag();
 213             if (!opt_int(opt_arg(), &single_iter))
 214                 goto end;
 215             break;
 216         case OPT_TEST_INDENT:
 217             if (!opt_int(opt_arg(), &value))
 218                 goto end;
 219             level = 4 * value;
 220             test_adjust_streams_tap_level(level);
 221             break;
 222         case OPT_TEST_SEED:
 223             if (!opt_int(opt_arg(), &value))
 224                 goto end;
 225             set_seed(value);
 226             break;
 227         }
 228     }
 229     if (!check_single_test_params(testname, flag_test, flag_iter))
 230         goto end;
 231     ret = 1;
 232 end:
 233     return ret;
 234 }
 235
 236
 237 int pulldown_test_framework(int ret)
 238 {
 239     set_test_title(NULL);
 240     return ret;
 241 }
 242
 243 static void finalize(int success)
 244 {
 245     if (success)
 246         ERR_clear_error();
 247     else
 248         ERR_print_errors_cb(openssl_error_cb, NULL);
 249 }
 250
 251 static char *test_title = NULL;
 252
 253 void set_test_title(const char *title)
 254 {
 255     free(test_title);
 256     test_title = title == NULL ? NULL : strdup(title);
 257 }
 258
 259 PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
 260                                              const char *description, ...)
 261 {
 262     va_list ap;
 263
 264     test_flush_stdout();
 265     test_flush_stderr();
 266
 267     if (verdict == 0 && seed != 0)
 268         test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed);
 269     test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok");
 270     va_start(ap, description);
 271     test_vprintf_tapout(description, ap);
 272     va_end(ap);
 273     if (verdict == TEST_SKIP_CODE)
 274         test_printf_tapout(" # skipped");
 275     test_printf_tapout("\n");
 276     test_flush_tapout();
 277 }
 278
 279 int run_tests(const char *test_prog_name)
 280 {
 281     int num_failed = 0;
 282     int verdict = 1;
 283     int ii, i, jj, j, jstep;
 284     int test_case_count = 0;
 285     int subtest_case_count = 0;
 286     int permute[OSSL_NELEM(all_tests)];
 287
 288     i = process_shared_options();
 289     if (i == 0)
 290         return EXIT_SUCCESS;
 291     if (i == -1)
 292         return EXIT_FAILURE;
 293
 294     if (num_tests < 1) {
 295         test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name);
 296     } else if (show_list == 0 && single_test == -1) {
 297         if (level > 0) {
 298             test_printf_stdout("Subtest: %s\n", test_prog_name);
 299             test_flush_stdout();
 300         }
 301         test_printf_tapout("1..%d\n", num_test_cases);
 302     }
 303
 304     test_flush_tapout();
 305
 306     for (i = 0; i < num_tests; i++)
 307         permute[i] = i;
 308     if (rand_order != 0)
 309         for (i = num_tests - 1; i >= 1; i--) {
 310             j = test_random() % (1 + i);
 311             ii = permute[j];
 312             permute[j] = permute[i];
 313             permute[i] = ii;
 314         }
 315
 316     for (ii = 0; ii != num_tests; ++ii) {
 317         i = permute[ii];
 318
 319         if (single_test != -1 && ((i+1) != single_test)) {
 320             continue;
 321         }
 322         else if (show_list) {
 323             if (all_tests[i].num != -1) {
 324                 test_printf_tapout("%d - %s (%d..%d)\n", ii + 1,
 325                                    all_tests[i].test_case_name, 1,
 326                                    all_tests[i].num);
 327             } else {
 328                 test_printf_tapout("%d - %s\n", ii + 1,
 329                                    all_tests[i].test_case_name);
 330             }
 331             test_flush_tapout();
 332         } else if (all_tests[i].num == -1) {
 333             set_test_title(all_tests[i].test_case_name);
 334             verdict = all_tests[i].test_fn();
 335             finalize(verdict != 0);
 336             test_verdict(verdict, "%d - %s", test_case_count + 1, test_title);
 337             if (verdict == 0)
 338                 num_failed++;
 339             test_case_count++;
 340         } else {
 341             int num_failed_inner = 0;
 342
 343             verdict = TEST_SKIP_CODE;
 344             set_test_title(all_tests[i].test_case_name);
 345             if (all_tests[i].subtest) {
 346                 level += 4;
 347                 test_adjust_streams_tap_level(level);
 348                 if (single_iter == -1) {
 349                     test_printf_stdout("Subtest: %s\n", test_title);
 350                     test_printf_tapout("%d..%d\n", 1, all_tests[i].num);
 351                     test_flush_stdout();
 352                     test_flush_tapout();
 353                 }
 354             }
 355
 356             j = -1;
 357             if (rand_order == 0 || all_tests[i].num < 3)
 358                 jstep = 1;
 359             else
 360                 do
 361                     jstep = test_random() % all_tests[i].num;
 362                 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
 363
 364             for (jj = 0; jj < all_tests[i].num; jj++) {
 365                 int v;
 366
 367                 j = (j + jstep) % all_tests[i].num;
 368                 if (single_iter != -1 && ((jj + 1) != single_iter))
 369                     continue;
 370                 v = all_tests[i].param_test_fn(j);
 371
 372                 if (v == 0) {
 373                     ++num_failed_inner;
 374                     verdict = 0;
 375                 } else if (v != TEST_SKIP_CODE && verdict != 0) {
 376                     verdict = 1;
 377                 }
 378
 379                 finalize(v != 0);
 380
 381                 if (all_tests[i].subtest)
 382                     test_verdict(v, "%d - iteration %d",
 383                                  subtest_case_count + 1, j + 1);
 384                 else
 385                     test_verdict(v, "%d - %s - iteration %d",
 386                                  test_case_count + subtest_case_count + 1,
 387                                  test_title, j + 1);
 388                 subtest_case_count++;
 389             }
 390
 391             if (all_tests[i].subtest) {
 392                 level -= 4;
 393                 test_adjust_streams_tap_level(level);
 394             }
 395             if (verdict == 0)
 396                 ++num_failed;
 397             if (all_tests[i].num == -1 || all_tests[i].subtest)
 398                 test_verdict(verdict, "%d - %s", test_case_count + 1,
 399                              all_tests[i].test_case_name);
 400             test_case_count++;
 401         }
 402     }
 403     if (num_failed != 0)
 404         return EXIT_FAILURE;
 405     return EXIT_SUCCESS;
 406 }
 407
 408 /*
 409  * Glue an array of strings together and return it as an allocated string.
 410  * Optionally return the whole length of this string in |out_len|
 411  */
 412 char *glue_strings(const char *list[], size_t *out_len)
 413 {
 414     size_t len = 0;
 415     char *p, *ret;
 416     int i;
 417
 418     for (i = 0; list[i] != NULL; i++)
 419         len += strlen(list[i]);
 420
 421     if (out_len != NULL)
 422         *out_len = len;
 423
 424     if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
 425         return NULL;
 426
 427     for (i = 0; list[i] != NULL; i++)
 428         p += strlen(strcpy(p, list[i]));
 429
 430     return ret;
 431 }
 432
 433 char *test_mk_file_path(const char *dir, const char *file)
 434 {
 435 # ifndef OPENSSL_SYS_VMS
 436     const char *sep = "/";
 437 # else
 438     const char *sep = "";
 439 # endif
 440     size_t len = strlen(dir) + strlen(sep) + strlen(file) + 1;
 441     char *full_file = OPENSSL_zalloc(len);
 442
 443     if (full_file != NULL) {
 444         OPENSSL_strlcpy(full_file, dir, len);
 445         OPENSSL_strlcat(full_file, sep, len);
 446         OPENSSL_strlcat(full_file, file, len);
 447     }
 448
 449     return full_file;
 450 }