test/threadstest.c

   1 /*
   2  * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved.
   3  *
   4  * Licensed under the Apache License 2.0 (the "License").  You may not use
   5  * this file except in compliance with the License.  You can obtain a copy
   6  * in the file LICENSE in the source distribution or at
   7  * https://www.openssl.org/source/license.html
   8  */
   9
  10 /*
  11  * The test_multi_downgrade_shared_pkey function tests the thread safety of a
  12  * deprecated function.
  13  */
  14 #ifndef OPENSSL_NO_DEPRECATED_3_0
  15 # define OPENSSL_SUPPRESS_DEPRECATED
  16 #endif
  17
  18 #if defined(_WIN32)
  19 # include <windows.h>
  20 #endif
  21
  22 #include <string.h>
  23 #include <openssl/crypto.h>
  24 #include <openssl/rsa.h>
  25 #include <openssl/aes.h>
  26 #include <openssl/err.h>
  27 #include "internal/tsan_assist.h"
  28 #include "internal/nelem.h"
  29 #include "testutil.h"
  30 #include "threadstest.h"
  31
  32 /* Limit the maximum number of threads */
  33 #define MAXIMUM_THREADS     3
  34
  35 /* Limit the maximum number of providers loaded into a library context */
  36 #define MAXIMUM_PROVIDERS   4
  37
  38 static int do_fips = 0;
  39 static char *privkey;
  40 static char *config_file = NULL;
  41 static int multidefault_run = 0;
  42
  43 static const char *default_provider[] = { "default", NULL };
  44 static const char *fips_provider[] = { "fips", NULL };
  45 static const char *fips_and_default_providers[] = { "default", "fips", NULL };
  46
  47 /* Grab a globally unique integer value */
  48 static int get_new_uid(void)
  49 {
  50     /*
  51      * Start with a nice large number to avoid potential conflicts when
  52      * we generate a new OID.
  53      */
  54     static TSAN_QUALIFIER int current_uid = 1 << (sizeof(int) * 8 - 2);
  55
  56     return tsan_counter(&current_uid);
  57 }
  58
  59 static int test_lock(void)
  60 {
  61     CRYPTO_RWLOCK *lock = CRYPTO_THREAD_lock_new();
  62     int res;
  63
  64     res = TEST_true(CRYPTO_THREAD_read_lock(lock))
  65           && TEST_true(CRYPTO_THREAD_unlock(lock));
  66
  67     CRYPTO_THREAD_lock_free(lock);
  68
  69     return res;
  70 }
  71
  72 static CRYPTO_ONCE once_run = CRYPTO_ONCE_STATIC_INIT;
  73 static unsigned once_run_count = 0;
  74
  75 static void once_do_run(void)
  76 {
  77     once_run_count++;
  78 }
  79
  80 static void once_run_thread_cb(void)
  81 {
  82     CRYPTO_THREAD_run_once(&once_run, once_do_run);
  83 }
  84
  85 static int test_once(void)
  86 {
  87     thread_t thread;
  88
  89     if (!TEST_true(run_thread(&thread, once_run_thread_cb))
  90         || !TEST_true(wait_for_thread(thread))
  91         || !CRYPTO_THREAD_run_once(&once_run, once_do_run)
  92         || !TEST_int_eq(once_run_count, 1))
  93         return 0;
  94     return 1;
  95 }
  96
  97 static CRYPTO_THREAD_LOCAL thread_local_key;
  98 static unsigned destructor_run_count = 0;
  99 static int thread_local_thread_cb_ok = 0;
 100
 101 static void thread_local_destructor(void *arg)
 102 {
 103     unsigned *count;
 104
 105     if (arg == NULL)
 106         return;
 107
 108     count = arg;
 109
 110     (*count)++;
 111 }
 112
 113 static void thread_local_thread_cb(void)
 114 {
 115     void *ptr;
 116
 117     ptr = CRYPTO_THREAD_get_local(&thread_local_key);
 118     if (!TEST_ptr_null(ptr)
 119         || !TEST_true(CRYPTO_THREAD_set_local(&thread_local_key,
 120                                               &destructor_run_count)))
 121         return;
 122
 123     ptr = CRYPTO_THREAD_get_local(&thread_local_key);
 124     if (!TEST_ptr_eq(ptr, &destructor_run_count))
 125         return;
 126
 127     thread_local_thread_cb_ok = 1;
 128 }
 129
 130 static int test_thread_local(void)
 131 {
 132     thread_t thread;
 133     void *ptr = NULL;
 134
 135     if (!TEST_true(CRYPTO_THREAD_init_local(&thread_local_key,
 136                                             thread_local_destructor)))
 137         return 0;
 138
 139     ptr = CRYPTO_THREAD_get_local(&thread_local_key);
 140     if (!TEST_ptr_null(ptr)
 141         || !TEST_true(run_thread(&thread, thread_local_thread_cb))
 142         || !TEST_true(wait_for_thread(thread))
 143         || !TEST_int_eq(thread_local_thread_cb_ok, 1))
 144         return 0;
 145
 146 #if defined(OPENSSL_THREADS) && !defined(CRYPTO_TDEBUG)
 147
 148     ptr = CRYPTO_THREAD_get_local(&thread_local_key);
 149     if (!TEST_ptr_null(ptr))
 150         return 0;
 151
 152 # if !defined(OPENSSL_SYS_WINDOWS)
 153     if (!TEST_int_eq(destructor_run_count, 1))
 154         return 0;
 155 # endif
 156 #endif
 157
 158     if (!TEST_true(CRYPTO_THREAD_cleanup_local(&thread_local_key)))
 159         return 0;
 160     return 1;
 161 }
 162
 163 static int test_atomic(void)
 164 {
 165     int val = 0, ret = 0, testresult = 0;
 166     uint64_t val64 = 1, ret64 = 0;
 167     CRYPTO_RWLOCK *lock = CRYPTO_THREAD_lock_new();
 168
 169     if (!TEST_ptr(lock))
 170         return 0;
 171
 172     if (CRYPTO_atomic_add(&val, 1, &ret, NULL)) {
 173         /* This succeeds therefore we're on a platform with lockless atomics */
 174         if (!TEST_int_eq(val, 1) || !TEST_int_eq(val, ret))
 175             goto err;
 176     } else {
 177         /* This failed therefore we're on a platform without lockless atomics */
 178         if (!TEST_int_eq(val, 0) || !TEST_int_eq(val, ret))
 179             goto err;
 180     }
 181     val = 0;
 182     ret = 0;
 183
 184     if (!TEST_true(CRYPTO_atomic_add(&val, 1, &ret, lock)))
 185         goto err;
 186     if (!TEST_int_eq(val, 1) || !TEST_int_eq(val, ret))
 187         goto err;
 188
 189     if (CRYPTO_atomic_or(&val64, 2, &ret64, NULL)) {
 190         /* This succeeds therefore we're on a platform with lockless atomics */
 191         if (!TEST_uint_eq((unsigned int)val64, 3)
 192                 || !TEST_uint_eq((unsigned int)val64, (unsigned int)ret64))
 193             goto err;
 194     } else {
 195         /* This failed therefore we're on a platform without lockless atomics */
 196         if (!TEST_uint_eq((unsigned int)val64, 1)
 197                 || !TEST_int_eq((unsigned int)ret64, 0))
 198             goto err;
 199     }
 200     val64 = 1;
 201     ret64 = 0;
 202
 203     if (!TEST_true(CRYPTO_atomic_or(&val64, 2, &ret64, lock)))
 204         goto err;
 205
 206     if (!TEST_uint_eq((unsigned int)val64, 3)
 207             || !TEST_uint_eq((unsigned int)val64, (unsigned int)ret64))
 208         goto err;
 209
 210     ret64 = 0;
 211     if (CRYPTO_atomic_load(&val64, &ret64, NULL)) {
 212         /* This succeeds therefore we're on a platform with lockless atomics */
 213         if (!TEST_uint_eq((unsigned int)val64, 3)
 214                 || !TEST_uint_eq((unsigned int)val64, (unsigned int)ret64))
 215             goto err;
 216     } else {
 217         /* This failed therefore we're on a platform without lockless atomics */
 218         if (!TEST_uint_eq((unsigned int)val64, 3)
 219                 || !TEST_int_eq((unsigned int)ret64, 0))
 220             goto err;
 221     }
 222
 223     ret64 = 0;
 224     if (!TEST_true(CRYPTO_atomic_load(&val64, &ret64, lock)))
 225         goto err;
 226
 227     if (!TEST_uint_eq((unsigned int)val64, 3)
 228             || !TEST_uint_eq((unsigned int)val64, (unsigned int)ret64))
 229         goto err;
 230
 231     testresult = 1;
 232  err:
 233     CRYPTO_THREAD_lock_free(lock);
 234     return testresult;
 235 }
 236
 237 static OSSL_LIB_CTX *multi_libctx = NULL;
 238 static int multi_success;
 239 static OSSL_PROVIDER *multi_provider[MAXIMUM_PROVIDERS + 1];
 240 static size_t multi_num_threads;
 241 static thread_t multi_threads[MAXIMUM_THREADS];
 242
 243 static void multi_intialise(void)
 244 {
 245     multi_success = 1;
 246     multi_libctx = NULL;
 247     multi_num_threads = 0;
 248     memset(multi_threads, 0, sizeof(multi_threads));
 249     memset(multi_provider, 0, sizeof(multi_provider));
 250 }
 251
 252 static void thead_teardown_libctx(void)
 253 {
 254     OSSL_PROVIDER **p;
 255
 256     for (p = multi_provider; *p != NULL; p++)
 257         OSSL_PROVIDER_unload(*p);
 258     OSSL_LIB_CTX_free(multi_libctx);
 259     multi_intialise();
 260 }
 261
 262 static int thread_setup_libctx(int libctx, const char *providers[])
 263 {
 264     size_t n;
 265
 266     if (libctx && !TEST_true(test_get_libctx(&multi_libctx, NULL, config_file,
 267                                              NULL, NULL)))
 268         return 0;
 269
 270     if (providers != NULL)
 271         for (n = 0; providers[n] != NULL; n++)
 272             if (!TEST_size_t_lt(n, MAXIMUM_PROVIDERS)
 273                 || !TEST_ptr(multi_provider[n] = OSSL_PROVIDER_load(multi_libctx,
 274                                                                     providers[n]))) {
 275                 thead_teardown_libctx();
 276                 return 0;
 277             }
 278     return 1;
 279 }
 280
 281 static int teardown_threads(void)
 282 {
 283     size_t i;
 284
 285     for (i = 0; i < multi_num_threads; i++)
 286         if (!TEST_true(wait_for_thread(multi_threads[i])))
 287             return 0;
 288     return 1;
 289 }
 290
 291 static int start_threads(size_t n, void (*thread_func)(void))
 292 {
 293     size_t i;
 294
 295     if (!TEST_size_t_le(multi_num_threads + n, MAXIMUM_THREADS))
 296         return 0;
 297
 298     for (i = 0 ; i < n; i++)
 299         if (!TEST_true(run_thread(multi_threads + multi_num_threads++, thread_func)))
 300             return 0;
 301     return 1;
 302 }
 303
 304 /* Template multi-threaded test function */
 305 static int thread_run_test(void (*main_func)(void),
 306                            size_t num_threads, void (*thread_func)(void),
 307                            int libctx, const char *providers[])
 308 {
 309     int testresult = 0;
 310
 311     multi_intialise();
 312     if (!thread_setup_libctx(libctx, providers)
 313             || !start_threads(num_threads, thread_func))
 314         goto err;
 315
 316     if (main_func != NULL)
 317         main_func();
 318
 319     if (!teardown_threads()
 320             || !TEST_true(multi_success))
 321         goto err;
 322     testresult = 1;
 323  err:
 324     thead_teardown_libctx();
 325     return testresult;
 326 }
 327
 328 static void thread_general_worker(void)
 329 {
 330     EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
 331     EVP_MD *md = EVP_MD_fetch(multi_libctx, "SHA2-256", NULL);
 332     EVP_CIPHER_CTX *cipherctx = EVP_CIPHER_CTX_new();
 333     EVP_CIPHER *ciph = EVP_CIPHER_fetch(multi_libctx, "AES-128-CBC", NULL);
 334     const char *message = "Hello World";
 335     size_t messlen = strlen(message);
 336     /* Should be big enough for encryption output too */
 337     unsigned char out[EVP_MAX_MD_SIZE];
 338     const unsigned char key[AES_BLOCK_SIZE] = {
 339         0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
 340         0x0c, 0x0d, 0x0e, 0x0f
 341     };
 342     const unsigned char iv[AES_BLOCK_SIZE] = {
 343         0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b,
 344         0x0c, 0x0d, 0x0e, 0x0f
 345     };
 346     unsigned int mdoutl;
 347     int ciphoutl;
 348     EVP_PKEY *pkey = NULL;
 349     int testresult = 0;
 350     int i, isfips;
 351
 352     isfips = OSSL_PROVIDER_available(multi_libctx, "fips");
 353
 354     if (!TEST_ptr(mdctx)
 355             || !TEST_ptr(md)
 356             || !TEST_ptr(cipherctx)
 357             || !TEST_ptr(ciph))
 358         goto err;
 359
 360     /* Do some work */
 361     for (i = 0; i < 5; i++) {
 362         if (!TEST_true(EVP_DigestInit_ex(mdctx, md, NULL))
 363                 || !TEST_true(EVP_DigestUpdate(mdctx, message, messlen))
 364                 || !TEST_true(EVP_DigestFinal(mdctx, out, &mdoutl)))
 365             goto err;
 366     }
 367     for (i = 0; i < 5; i++) {
 368         if (!TEST_true(EVP_EncryptInit_ex(cipherctx, ciph, NULL, key, iv))
 369                 || !TEST_true(EVP_EncryptUpdate(cipherctx, out, &ciphoutl,
 370                                                 (unsigned char *)message,
 371                                                 messlen))
 372                 || !TEST_true(EVP_EncryptFinal(cipherctx, out, &ciphoutl)))
 373             goto err;
 374     }
 375
 376     /*
 377      * We want the test to run quickly - not securely.
 378      * Therefore we use an insecure bit length where we can (512).
 379      * In the FIPS module though we must use a longer length.
 380      */
 381     pkey = EVP_PKEY_Q_keygen(multi_libctx, NULL, "RSA", isfips ? 2048 : 512);
 382     if (!TEST_ptr(pkey))
 383         goto err;
 384
 385     testresult = 1;
 386  err:
 387     EVP_MD_CTX_free(mdctx);
 388     EVP_MD_free(md);
 389     EVP_CIPHER_CTX_free(cipherctx);
 390     EVP_CIPHER_free(ciph);
 391     EVP_PKEY_free(pkey);
 392     if (!testresult)
 393         multi_success = 0;
 394 }
 395
 396 static void thread_multi_simple_fetch(void)
 397 {
 398     EVP_MD *md = EVP_MD_fetch(multi_libctx, "SHA2-256", NULL);
 399
 400     if (md != NULL)
 401         EVP_MD_free(md);
 402     else
 403         multi_success = 0;
 404 }
 405
 406 static EVP_PKEY *shared_evp_pkey = NULL;
 407
 408 static void thread_shared_evp_pkey(void)
 409 {
 410     char *msg = "Hello World";
 411     unsigned char ctbuf[256];
 412     unsigned char ptbuf[256];
 413     size_t ptlen = sizeof(ptbuf), ctlen = sizeof(ctbuf);
 414     EVP_PKEY_CTX *ctx = NULL;
 415     int success = 0;
 416     int i;
 417
 418     for (i = 0; i < 1 + do_fips; i++) {
 419         if (i > 0)
 420             EVP_PKEY_CTX_free(ctx);
 421         ctx = EVP_PKEY_CTX_new_from_pkey(multi_libctx, shared_evp_pkey,
 422                                          i == 0 ? "provider=default"
 423                                                 : "provider=fips");
 424         if (!TEST_ptr(ctx))
 425             goto err;
 426
 427         if (!TEST_int_ge(EVP_PKEY_encrypt_init(ctx), 0)
 428                 || !TEST_int_ge(EVP_PKEY_encrypt(ctx, ctbuf, &ctlen,
 429                                                 (unsigned char *)msg, strlen(msg)),
 430                                                 0))
 431             goto err;
 432
 433         EVP_PKEY_CTX_free(ctx);
 434         ctx = EVP_PKEY_CTX_new_from_pkey(multi_libctx, shared_evp_pkey, NULL);
 435
 436         if (!TEST_ptr(ctx))
 437             goto err;
 438
 439         if (!TEST_int_ge(EVP_PKEY_decrypt_init(ctx), 0)
 440                 || !TEST_int_ge(EVP_PKEY_decrypt(ctx, ptbuf, &ptlen, ctbuf, ctlen),
 441                                                 0)
 442                 || !TEST_mem_eq(msg, strlen(msg), ptbuf, ptlen))
 443             goto err;
 444     }
 445
 446     success = 1;
 447
 448  err:
 449     EVP_PKEY_CTX_free(ctx);
 450     if (!success)
 451         multi_success = 0;
 452 }
 453
 454 static void thread_provider_load_unload(void)
 455 {
 456     OSSL_PROVIDER *deflt = OSSL_PROVIDER_load(multi_libctx, "default");
 457
 458     if (!TEST_ptr(deflt)
 459             || !TEST_true(OSSL_PROVIDER_available(multi_libctx, "default")))
 460         multi_success = 0;
 461
 462     OSSL_PROVIDER_unload(deflt);
 463 }
 464
 465 static int test_multi_general_worker_default_provider(void)
 466 {
 467     return thread_run_test(&thread_general_worker, 2, &thread_general_worker,
 468                            1, default_provider);
 469 }
 470
 471 static int test_multi_general_worker_fips_provider(void)
 472 {
 473     if (!do_fips)
 474         return TEST_skip("FIPS not supported");
 475     return thread_run_test(&thread_general_worker, 2, &thread_general_worker,
 476                            1, fips_provider);
 477 }
 478
 479 static int test_multi_fetch_worker(void)
 480 {
 481     return thread_run_test(&thread_multi_simple_fetch,
 482                            2, &thread_multi_simple_fetch, 1, default_provider);
 483 }
 484
 485 static int test_multi_shared_pkey_common(void (*worker)(void))
 486 {
 487     int testresult = 0;
 488
 489     multi_intialise();
 490     if (!thread_setup_libctx(1, do_fips ? fips_and_default_providers
 491                                         : default_provider)
 492             || !TEST_ptr(shared_evp_pkey = load_pkey_pem(privkey, multi_libctx))
 493             || !start_threads(1, &thread_shared_evp_pkey)
 494             || !start_threads(1, worker))
 495         goto err;
 496
 497     thread_shared_evp_pkey();
 498
 499     if (!teardown_threads()
 500             || !TEST_true(multi_success))
 501         goto err;
 502     testresult = 1;
 503  err:
 504     EVP_PKEY_free(shared_evp_pkey);
 505     thead_teardown_libctx();
 506     return testresult;
 507 }
 508
 509 #ifndef OPENSSL_NO_DEPRECATED_3_0
 510 static void thread_downgrade_shared_evp_pkey(void)
 511 {
 512     /*
 513      * This test is only relevant for deprecated functions that perform
 514      * downgrading
 515      */
 516     if (EVP_PKEY_get0_RSA(shared_evp_pkey) == NULL)
 517         multi_success = 0;
 518 }
 519
 520 static int test_multi_downgrade_shared_pkey(void)
 521 {
 522     return test_multi_shared_pkey_common(&thread_downgrade_shared_evp_pkey);
 523 }
 524 #endif
 525
 526 static int test_multi_shared_pkey(void)
 527 {
 528     return test_multi_shared_pkey_common(&thread_shared_evp_pkey);
 529 }
 530
 531 static int test_multi_load_unload_provider(void)
 532 {
 533     EVP_MD *sha256 = NULL;
 534     OSSL_PROVIDER *prov = NULL;
 535     int testresult = 0;
 536
 537     multi_intialise();
 538     if (!thread_setup_libctx(1, NULL)
 539             || !TEST_ptr(prov = OSSL_PROVIDER_load(multi_libctx, "default"))
 540             || !TEST_ptr(sha256 = EVP_MD_fetch(multi_libctx, "SHA2-256", NULL))
 541             || !TEST_true(OSSL_PROVIDER_unload(prov)))
 542         goto err;
 543     prov = NULL;
 544
 545     if (!start_threads(2, &thread_provider_load_unload))
 546         goto err;
 547
 548     thread_provider_load_unload();
 549
 550     if (!teardown_threads()
 551             || !TEST_true(multi_success))
 552         goto err;
 553     testresult = 1;
 554  err:
 555     OSSL_PROVIDER_unload(prov);
 556     EVP_MD_free(sha256);
 557     thead_teardown_libctx();
 558     return testresult;
 559 }
 560
 561 /*
 562  * This test attempts to load several providers at the same time, and if
 563  * run with a thread sanitizer, should crash if the core provider code
 564  * doesn't synchronize well enough.
 565  */
 566 static void test_multi_load_worker(void)
 567 {
 568     OSSL_PROVIDER *prov;
 569
 570     if (!TEST_ptr(prov = OSSL_PROVIDER_load(multi_libctx, "default"))
 571             || !TEST_true(OSSL_PROVIDER_unload(prov)))
 572         multi_success = 0;
 573 }
 574
 575 static int test_multi_default(void)
 576 {
 577     /* Avoid running this test twice */
 578     if (multidefault_run) {
 579         TEST_skip("multi default test already run");
 580         return 1;
 581     }
 582     multidefault_run = 1;
 583
 584     return thread_run_test(&thread_multi_simple_fetch,
 585                            2, &thread_multi_simple_fetch, 0, default_provider);
 586 }
 587
 588 static int test_multi_load(void)
 589 {
 590     int res = 1;
 591
 592     /* The multidefault test must run prior to this test */
 593     if (!multidefault_run) {
 594         TEST_info("Running multi default test first");
 595         res = test_multi_default();
 596     }
 597
 598     return thread_run_test(NULL, 3, &test_multi_load_worker, 0, NULL) && res;
 599 }
 600
 601 static void test_obj_create_one(void)
 602 {
 603     char tids[12], oid[40], sn[30], ln[30];
 604     int id = get_new_uid();
 605
 606     BIO_snprintf(tids, sizeof(tids), "%d", id);
 607     BIO_snprintf(oid, sizeof(oid), "1.3.6.1.4.1.16604.%s", tids);
 608     BIO_snprintf(sn, sizeof(sn), "short-name-%s", tids);
 609     BIO_snprintf(ln, sizeof(ln), "long-name-%s", tids);
 610     if (!TEST_true(id = OBJ_create(oid, sn, ln))
 611             || !TEST_true(OBJ_add_sigid(id, NID_sha3_256, NID_rsa)))
 612         multi_success = 0;
 613 }
 614
 615 static int test_obj_add(void)
 616 {
 617     return thread_run_test(&test_obj_create_one,
 618                            MAXIMUM_THREADS, &test_obj_create_one,
 619                            1, default_provider);
 620 }
 621
 622 typedef enum OPTION_choice {
 623     OPT_ERR = -1,
 624     OPT_EOF = 0,
 625     OPT_FIPS, OPT_CONFIG_FILE,
 626     OPT_TEST_ENUM
 627 } OPTION_CHOICE;
 628
 629 const OPTIONS *test_get_options(void)
 630 {
 631     static const OPTIONS options[] = {
 632         OPT_TEST_OPTIONS_DEFAULT_USAGE,
 633         { "fips", OPT_FIPS, '-', "Test the FIPS provider" },
 634         { "config", OPT_CONFIG_FILE, '<',
 635           "The configuration file to use for the libctx" },
 636         { NULL }
 637     };
 638     return options;
 639 }
 640
 641 int setup_tests(void)
 642 {
 643     OPTION_CHOICE o;
 644     char *datadir;
 645
 646     while ((o = opt_next()) != OPT_EOF) {
 647         switch (o) {
 648         case OPT_FIPS:
 649             do_fips = 1;
 650             break;
 651         case OPT_CONFIG_FILE:
 652             config_file = opt_arg();
 653             break;
 654         case OPT_TEST_CASES:
 655             break;
 656         default:
 657             return 0;
 658         }
 659     }
 660
 661     if (!TEST_ptr(datadir = test_get_argument(0)))
 662         return 0;
 663
 664     privkey = test_mk_file_path(datadir, "rsakey.pem");
 665     if (!TEST_ptr(privkey))
 666         return 0;
 667
 668     /* Keep first to validate auto creation of default library context */
 669     ADD_TEST(test_multi_default);
 670
 671     ADD_TEST(test_lock);
 672     ADD_TEST(test_once);
 673     ADD_TEST(test_thread_local);
 674     ADD_TEST(test_atomic);
 675     ADD_TEST(test_multi_load);
 676     ADD_TEST(test_multi_general_worker_default_provider);
 677     ADD_TEST(test_multi_general_worker_fips_provider);
 678     ADD_TEST(test_multi_fetch_worker);
 679     ADD_TEST(test_multi_shared_pkey);
 680 #ifndef OPENSSL_NO_DEPRECATED_3_0
 681     ADD_TEST(test_multi_downgrade_shared_pkey);
 682 #endif
 683     ADD_TEST(test_multi_load_unload_provider);
 684     ADD_TEST(test_obj_add);
 685     return 1;
 686 }
 687
 688 void cleanup_tests(void)
 689 {
 690     OPENSSL_free(privkey);
 691 }