test/testutil/driver.c

   1 /*
   2  * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License 2.0 (the "License").  You may not use
   5  * this file except in compliance with the License.  You can obtain a copy
   6  * in the file LICENSE in the source distribution or at
   7  * https://www.openssl.org/source/license.html
   8  */
   9
  10 #include "../testutil.h"
  11 #include "output.h"
  12 #include "tu_local.h"
  13
  14 #include <string.h>
  15 #include <assert.h>
  16
  17 #include "internal/nelem.h"
  18 #include <openssl/bio.h>
  19
  20 #ifdef _WIN32
  21 # define strdup _strdup
  22 #endif
  23
  24
  25 /*
  26  * Declares the structures needed to register each test case function.
  27  */
  28 typedef struct test_info {
  29     const char *test_case_name;
  30     int (*test_fn) (void);
  31     int (*param_test_fn)(int idx);
  32     int num;
  33
  34     /* flags */
  35     int subtest:1;
  36 } TEST_INFO;
  37
  38 static TEST_INFO all_tests[1024];
  39 static int num_tests = 0;
  40 static int show_list = 0;
  41 static int single_test = -1;
  42 static int single_iter = -1;
  43 static int level = 0;
  44 static int seed = 0;
  45 /*
  46  * A parameterised test runs a loop of test cases.
  47  * |num_test_cases| counts the total number of test cases
  48  * across all tests.
  49  */
  50 static int num_test_cases = 0;
  51
  52 static int process_shared_options(void);
  53
  54
  55 void add_test(const char *test_case_name, int (*test_fn) (void))
  56 {
  57     assert(num_tests != OSSL_NELEM(all_tests));
  58     all_tests[num_tests].test_case_name = test_case_name;
  59     all_tests[num_tests].test_fn = test_fn;
  60     all_tests[num_tests].num = -1;
  61     ++num_tests;
  62     ++num_test_cases;
  63 }
  64
  65 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
  66                    int num, int subtest)
  67 {
  68     assert(num_tests != OSSL_NELEM(all_tests));
  69     all_tests[num_tests].test_case_name = test_case_name;
  70     all_tests[num_tests].param_test_fn = test_fn;
  71     all_tests[num_tests].num = num;
  72     all_tests[num_tests].subtest = subtest;
  73     ++num_tests;
  74     num_test_cases += num;
  75 }
  76
  77 int subtest_level(void)
  78 {
  79     return level;
  80 }
  81
  82 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
  83 static int should_report_leaks(void)
  84 {
  85     /*
  86      * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
  87      * can be used to disable leak checking at runtime.
  88      * Note this only works when running the test binary manually;
  89      * the test harness always enables OPENSSL_DEBUG_MEMORY.
  90      */
  91     char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
  92
  93     return mem_debug_env == NULL
  94         || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
  95 }
  96 #endif
  97
  98 static int gcd(int a, int b)
  99 {
 100     while (b != 0) {
 101         int t = b;
 102         b = a % b;
 103         a = t;
 104     }
 105     return a;
 106 }
 107
 108 static void set_seed(int s)
 109 {
 110     seed = s;
 111     if (seed <= 0)
 112         seed = (int)time(NULL);
 113     test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);
 114     test_flush_stdout();
 115     srand(seed);
 116 }
 117
 118
 119 int setup_test_framework(int argc, char *argv[])
 120 {
 121     char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
 122     char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
 123
 124     if (TAP_levels != NULL)
 125         level = 4 * atoi(TAP_levels);
 126     if (test_seed != NULL)
 127         set_seed(atoi(test_seed));
 128
 129 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
 130     if (should_report_leaks()) {
 131         CRYPTO_set_mem_debug(1);
 132         CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
 133     }
 134 #endif
 135     if (!opt_init(argc, argv, test_get_options()))
 136         return 0;
 137     return 1;
 138 }
 139
 140
 141 /*
 142  * This can only be called after setup() has run, since num_tests and
 143  * all_tests[] are setup at this point
 144  */
 145 static int check_single_test_params(char *name, char *testname, char *itname)
 146 {
 147     if (name != NULL) {
 148         int i;
 149         for (i = 0; i < num_tests; ++i) {
 150             if (strcmp(name, all_tests[i].test_case_name) == 0) {
 151                 single_test = 1 + i;
 152                 break;
 153             }
 154         }
 155         if (i >= num_tests)
 156             single_test = atoi(name);
 157     }
 158
 159
 160     /* if only iteration is specified, assume we want the first test */
 161     if (single_test == -1 && single_iter != -1)
 162         single_test = 1;
 163
 164     if (single_test != -1) {
 165         if (single_test < 1 || single_test > num_tests) {
 166             test_printf_stderr("Invalid -%s value "
 167                                "(Value must be a valid test name OR a value between %d..%d)\n",
 168                                testname, 1, num_tests);
 169             return 0;
 170         }
 171     }
 172     if (single_iter != -1) {
 173         if (all_tests[single_test - 1].num == -1) {
 174             test_printf_stderr("-%s option is not valid for test %d:%s\n",
 175                                itname,
 176                                single_test,
 177                                all_tests[single_test - 1].test_case_name);
 178             return 0;
 179         } else if (single_iter < 1
 180                    || single_iter > all_tests[single_test - 1].num) {
 181             test_printf_stderr("Invalid -%s value for test %d:%s\t"
 182                                "(Value must be in the range %d..%d)\n",
 183                                itname, single_test,
 184                                all_tests[single_test - 1].test_case_name,
 185                                1, all_tests[single_test - 1].num);
 186             return 0;
 187         }
 188     }
 189     return 1;
 190 }
 191
 192 static int process_shared_options(void)
 193 {
 194     OPTION_CHOICE_DEFAULT o;
 195     int value;
 196     int ret = -1;
 197     char *flag_test = "";
 198     char *flag_iter = "";
 199     char *testname = NULL;
 200
 201     opt_begin();
 202     while ((o = opt_next()) != OPT_EOF) {
 203         switch (o) {
 204         /* Ignore any test options at this level */
 205         default:
 206             break;
 207         case OPT_ERR:
 208             return ret;
 209         case OPT_TEST_HELP:
 210             opt_help(test_get_options());
 211             return 0;
 212         case OPT_TEST_LIST:
 213             show_list = 1;
 214             break;
 215         case OPT_TEST_SINGLE:
 216             flag_test = opt_flag();
 217             testname = opt_arg();
 218             break;
 219         case OPT_TEST_ITERATION:
 220             flag_iter = opt_flag();
 221             if (!opt_int(opt_arg(), &single_iter))
 222                 goto end;
 223             break;
 224         case OPT_TEST_INDENT:
 225             if (!opt_int(opt_arg(), &value))
 226                 goto end;
 227             level = 4 * value;
 228             break;
 229         case OPT_TEST_SEED:
 230             if (!opt_int(opt_arg(), &value))
 231                 goto end;
 232             set_seed(value);
 233             break;
 234         }
 235     }
 236     if (!check_single_test_params(testname, flag_test, flag_iter))
 237         goto end;
 238     ret = 1;
 239 end:
 240     return ret;
 241 }
 242
 243
 244 int pulldown_test_framework(int ret)
 245 {
 246     set_test_title(NULL);
 247 #ifndef OPENSSL_NO_CRYPTO_MDEBUG
 248     if (should_report_leaks()
 249         && CRYPTO_mem_leaks_cb(openssl_error_cb, NULL) <= 0)
 250         return EXIT_FAILURE;
 251 #endif
 252     return ret;
 253 }
 254
 255 static void finalize(int success)
 256 {
 257     if (success)
 258         ERR_clear_error();
 259     else
 260         ERR_print_errors_cb(openssl_error_cb, NULL);
 261 }
 262
 263 static char *test_title = NULL;
 264
 265 void set_test_title(const char *title)
 266 {
 267     free(test_title);
 268     test_title = title == NULL ? NULL : strdup(title);
 269 }
 270
 271 PRINTF_FORMAT(2, 3) static void test_verdict(int pass, const char *extra, ...)
 272 {
 273     va_list ap;
 274
 275     test_flush_stdout();
 276     test_flush_stderr();
 277
 278     test_printf_stdout("%*s%s", level, "", pass ? "ok" : "not ok");
 279     if (extra != NULL) {
 280         test_printf_stdout(" ");
 281         va_start(ap, extra);
 282         test_vprintf_stdout(extra, ap);
 283         va_end(ap);
 284     }
 285     test_printf_stdout("\n");
 286     test_flush_stdout();
 287 }
 288
 289 int run_tests(const char *test_prog_name)
 290 {
 291     int num_failed = 0;
 292     int verdict = 1;
 293     int ii, i, jj, j, jstep;
 294     int permute[OSSL_NELEM(all_tests)];
 295
 296     i = process_shared_options();
 297     if (i == 0)
 298         return EXIT_SUCCESS;
 299     if (i == -1)
 300         return EXIT_FAILURE;
 301
 302     if (num_tests < 1) {
 303         test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",
 304                            test_prog_name);
 305     } else if (show_list == 0 && single_test == -1) {
 306         if (level > 0)
 307             test_printf_stdout("%*s# Subtest: %s\n", level, "", test_prog_name);
 308         test_printf_stdout("%*s1..%d\n", level, "", num_tests);
 309     }
 310
 311     test_flush_stdout();
 312
 313     for (i = 0; i < num_tests; i++)
 314         permute[i] = i;
 315     if (seed != 0)
 316         for (i = num_tests - 1; i >= 1; i--) {
 317             j = rand() % (1 + i);
 318             ii = permute[j];
 319             permute[j] = permute[i];
 320             permute[i] = ii;
 321         }
 322
 323     for (ii = 0; ii != num_tests; ++ii) {
 324         i = permute[ii];
 325
 326         if (single_test != -1 && ((i+1) != single_test)) {
 327             continue;
 328         }
 329         else if (show_list) {
 330             if (all_tests[i].num != -1) {
 331                 test_printf_stdout("%d - %s (%d..%d)\n", ii + 1,
 332                                    all_tests[i].test_case_name, 1,
 333                                    all_tests[i].num);
 334             } else {
 335                 test_printf_stdout("%d - %s\n", ii + 1,
 336                                    all_tests[i].test_case_name);
 337             }
 338             test_flush_stdout();
 339         } else if (all_tests[i].num == -1) {
 340             int ret = 0;
 341
 342             set_test_title(all_tests[i].test_case_name);
 343             ret = all_tests[i].test_fn();
 344             verdict = 1;
 345             if (!ret) {
 346                 verdict = 0;
 347                 ++num_failed;
 348             }
 349             test_verdict(verdict, "%d - %s", ii + 1, test_title);
 350             finalize(ret);
 351         } else {
 352             int num_failed_inner = 0;
 353
 354             level += 4;
 355             if (all_tests[i].subtest && single_iter == -1) {
 356                 test_printf_stdout("%*s# Subtest: %s\n", level, "",
 357                                    all_tests[i].test_case_name);
 358                 test_printf_stdout("%*s%d..%d\n", level, "", 1,
 359                                    all_tests[i].num);
 360                 test_flush_stdout();
 361             }
 362
 363             j = -1;
 364             if (seed == 0 || all_tests[i].num < 3)
 365                 jstep = 1;
 366             else
 367                 do
 368                     jstep = rand() % all_tests[i].num;
 369                 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
 370
 371             for (jj = 0; jj < all_tests[i].num; jj++) {
 372                 int ret;
 373
 374                 j = (j + jstep) % all_tests[i].num;
 375                 if (single_iter != -1 && ((jj + 1) != single_iter))
 376                     continue;
 377                 set_test_title(NULL);
 378                 ret = all_tests[i].param_test_fn(j);
 379
 380                 if (!ret)
 381                     ++num_failed_inner;
 382
 383                 finalize(ret);
 384
 385                 if (all_tests[i].subtest) {
 386                     verdict = 1;
 387                     if (!ret) {
 388                         verdict = 0;
 389                         ++num_failed_inner;
 390                     }
 391                     if (test_title != NULL)
 392                         test_verdict(verdict, "%d - %s", jj + 1, test_title);
 393                     else
 394                         test_verdict(verdict, "%d - iteration %d",
 395                                      jj + 1, j + 1);
 396                 }
 397             }
 398
 399             level -= 4;
 400             verdict = 1;
 401             if (num_failed_inner) {
 402                 verdict = 0;
 403                 ++num_failed;
 404             }
 405             test_verdict(verdict, "%d - %s", ii + 1,
 406                          all_tests[i].test_case_name);
 407         }
 408     }
 409     if (num_failed != 0)
 410         return EXIT_FAILURE;
 411     return EXIT_SUCCESS;
 412 }
 413
 414 /*
 415  * Glue an array of strings together and return it as an allocated string.
 416  * Optionally return the whole length of this string in |out_len|
 417  */
 418 char *glue_strings(const char *list[], size_t *out_len)
 419 {
 420     size_t len = 0;
 421     char *p, *ret;
 422     int i;
 423
 424     for (i = 0; list[i] != NULL; i++)
 425         len += strlen(list[i]);
 426
 427     if (out_len != NULL)
 428         *out_len = len;
 429
 430     if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
 431         return NULL;
 432
 433     for (i = 0; list[i] != NULL; i++)
 434         p += strlen(strcpy(p, list[i]));
 435
 436     return ret;
 437 }
 438