util/perl/OpenSSL/Test.pm

   1 # Copyright 2016-2019 The OpenSSL Project Authors. All Rights Reserved.
   2 #
   3 # Licensed under the Apache License 2.0 (the "License").  You may not use
   4 # this file except in compliance with the License.  You can obtain a copy
   5 # in the file LICENSE in the source distribution or at
   6 # https://www.openssl.org/source/license.html
   7
   8 package OpenSSL::Test;
   9
  10 use strict;
  11 use warnings;
  12
  13 use Test::More 0.96;
  14
  15 use Exporter;
  16 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
  17 $VERSION = "1.0";
  18 @ISA = qw(Exporter);
  19 @EXPORT = (@Test::More::EXPORT, qw(setup run indir cmd app fuzz test
  20                                    perlapp perltest subtest));
  21 @EXPORT_OK = (@Test::More::EXPORT_OK, qw(bldtop_dir bldtop_file
  22                                          srctop_dir srctop_file
  23                                          data_file data_dir
  24                                          pipe with cmdstr quotify
  25                                          openssl_versions
  26                                          ok_nofips is_nofips isnt_nofips));
  27
  28 =head1 NAME
  29
  30 OpenSSL::Test - a private extension of Test::More
  31
  32 =head1 SYNOPSIS
  33
  34   use OpenSSL::Test;
  35
  36   setup("my_test_name");
  37
  38   ok(run(app(["openssl", "version"])), "check for openssl presence");
  39
  40   indir "subdir" => sub {
  41     ok(run(test(["sometest", "arg1"], stdout => "foo.txt")),
  42        "run sometest with output to foo.txt");
  43   };
  44
  45 =head1 DESCRIPTION
  46
  47 This module is a private extension of L<Test::More> for testing OpenSSL.
  48 In addition to the Test::More functions, it also provides functions that
  49 easily find the diverse programs within a OpenSSL build tree, as well as
  50 some other useful functions.
  51
  52 This module I<depends> on the environment variables C<$TOP> or C<$SRCTOP>
  53 and C<$BLDTOP>.  Without one of the combinations it refuses to work.
  54 See L</ENVIRONMENT> below.
  55
  56 With each test recipe, a parallel data directory with (almost) the same name
  57 as the recipe is possible in the source directory tree.  For example, for a
  58 recipe C<$SRCTOP/test/recipes/99-foo.t>, there could be a directory
  59 C<$SRCTOP/test/recipes/99-foo_data/>.
  60
  61 =cut
  62
  63 use File::Copy;
  64 use File::Spec::Functions qw/file_name_is_absolute curdir canonpath splitdir
  65                              catdir catfile splitpath catpath devnull abs2rel
  66                              rel2abs/;
  67 use File::Path 2.00 qw/rmtree mkpath/;
  68 use File::Basename;
  69 use Cwd qw/abs_path/;
  70
  71 my $level = 0;
  72
  73 # The name of the test.  This is set by setup() and is used in the other
  74 # functions to verify that setup() has been used.
  75 my $test_name = undef;
  76
  77 # Directories we want to keep track of TOP, APPS, TEST and RESULTS are the
  78 # ones we're interested in, corresponding to the environment variables TOP
  79 # (mandatory), BIN_D, TEST_D, UTIL_D and RESULT_D.
  80 my %directories = ();
  81
  82 # The environment variables that gave us the contents in %directories.  These
  83 # get modified whenever we change directories, so that subprocesses can use
  84 # the values of those environment variables as well
  85 my @direnv = ();
  86
  87 # A bool saying if we shall stop all testing if the current recipe has failing
  88 # tests or not.  This is set by setup() if the environment variable STOPTEST
  89 # is defined with a non-empty value.
  90 my $end_with_bailout = 0;
  91
  92 # A set of hooks that is affected by with() and may be used in diverse places.
  93 # All hooks are expected to be CODE references.
  94 my %hooks = (
  95
  96     # exit_checker is used by run() directly after completion of a command.
  97     # it receives the exit code from that command and is expected to return
  98     # 1 (for success) or 0 (for failure).  This is the status value that run()
  99     # will give back (through the |statusvar| reference and as returned value
 100     # when capture => 1 doesn't apply).
 101     exit_checker => sub { return shift == 0 ? 1 : 0 },
 102
 103     );
 104
 105 # Debug flag, to be set manually when needed
 106 my $debug = 0;
 107
 108 =head2 Main functions
 109
 110 The following functions are exported by default when using C<OpenSSL::Test>.
 111
 112 =cut
 113
 114 =over 4
 115
 116 =item B<setup "NAME">
 117
 118 C<setup> is used for initial setup, and it is mandatory that it's used.
 119 If it's not used in a OpenSSL test recipe, the rest of the recipe will
 120 most likely refuse to run.
 121
 122 C<setup> checks for environment variables (see L</ENVIRONMENT> below),
 123 checks that C<$TOP/Configure> or C<$SRCTOP/Configure> exists, C<chdir>
 124 into the results directory (defined by the C<$RESULT_D> environment
 125 variable if defined, otherwise C<$BLDTOP/test> or C<$TOP/test>, whichever
 126 is defined).
 127
 128 =back
 129
 130 =cut
 131
 132 sub setup {
 133     my $old_test_name = $test_name;
 134     $test_name = shift;
 135
 136     BAIL_OUT("setup() must receive a name") unless $test_name;
 137     warn "setup() detected test name change.  Innocuous, so we continue...\n"
 138         if $old_test_name && $old_test_name ne $test_name;
 139
 140     return if $old_test_name;
 141
 142     BAIL_OUT("setup() needs \$TOP or \$SRCTOP and \$BLDTOP to be defined")
 143         unless $ENV{TOP} || ($ENV{SRCTOP} && $ENV{BLDTOP});
 144     BAIL_OUT("setup() found both \$TOP and \$SRCTOP or \$BLDTOP...")
 145         if $ENV{TOP} && ($ENV{SRCTOP} || $ENV{BLDTOP});
 146
 147     __env();
 148
 149     BAIL_OUT("setup() expects the file Configure in the source top directory")
 150         unless -f srctop_file("Configure");
 151
 152     __cwd($directories{RESULTS});
 153 }
 154
 155 =over 4
 156
 157 =item B<indir "SUBDIR" =E<gt> sub BLOCK, OPTS>
 158
 159 C<indir> is used to run a part of the recipe in a different directory than
 160 the one C<setup> moved into, usually a subdirectory, given by SUBDIR.
 161 The part of the recipe that's run there is given by the codeblock BLOCK.
 162
 163 C<indir> takes some additional options OPTS that affect the subdirectory:
 164
 165 =over 4
 166
 167 =item B<create =E<gt> 0|1>
 168
 169 When set to 1 (or any value that perl perceives as true), the subdirectory
 170 will be created if it doesn't already exist.  This happens before BLOCK
 171 is executed.
 172
 173 =item B<cleanup =E<gt> 0|1>
 174
 175 When set to 1 (or any value that perl perceives as true), the subdirectory
 176 will be cleaned out and removed.  This happens both before and after BLOCK
 177 is executed.
 178
 179 =back
 180
 181 An example:
 182
 183   indir "foo" => sub {
 184       ok(run(app(["openssl", "version"]), stdout => "foo.txt"));
 185       if (ok(open(RESULT, "foo.txt"), "reading foo.txt")) {
 186           my $line = <RESULT>;
 187           close RESULT;
 188           is($line, qr/^OpenSSL 1\./,
 189              "check that we're using OpenSSL 1.x.x");
 190       }
 191   }, create => 1, cleanup => 1;
 192
 193 =back
 194
 195 =cut
 196
 197 sub indir {
 198     my $subdir = shift;
 199     my $codeblock = shift;
 200     my %opts = @_;
 201
 202     my $reverse = __cwd($subdir,%opts);
 203     BAIL_OUT("FAILURE: indir, \"$subdir\" wasn't possible to move into")
 204         unless $reverse;
 205
 206     $codeblock->();
 207
 208     __cwd($reverse);
 209
 210     if ($opts{cleanup}) {
 211         rmtree($subdir, { safe => 0 });
 212     }
 213 }
 214
 215 =over 4
 216
 217 =item B<cmd ARRAYREF, OPTS>
 218
 219 This functions build up a platform dependent command based on the
 220 input.  It takes a reference to a list that is the executable or
 221 script and its arguments, and some additional options (described
 222 further on).  Where necessary, the command will be wrapped in a
 223 suitable environment to make sure the correct shared libraries are
 224 used (currently only on Unix).
 225
 226 It returns a CODEREF to be used by C<run>, C<pipe> or C<cmdstr>.
 227
 228 The options that C<cmd> can take are in the form of hash values:
 229
 230 =over 4
 231
 232 =item B<stdin =E<gt> PATH>
 233
 234 =item B<stdout =E<gt> PATH>
 235
 236 =item B<stderr =E<gt> PATH>
 237
 238 In all three cases, the corresponding standard input, output or error is
 239 redirected from (for stdin) or to (for the others) a file given by the
 240 string PATH, I<or>, if the value is C<undef>, C</dev/null> or similar.
 241
 242 =back
 243
 244 =item B<app ARRAYREF, OPTS>
 245
 246 =item B<test ARRAYREF, OPTS>
 247
 248 Both of these are specific applications of C<cmd>, with just a couple
 249 of small difference:
 250
 251 C<app> expects to find the given command (the first item in the given list
 252 reference) as an executable in C<$BIN_D> (if defined, otherwise C<$TOP/apps>
 253 or C<$BLDTOP/apps>).
 254
 255 C<test> expects to find the given command (the first item in the given list
 256 reference) as an executable in C<$TEST_D> (if defined, otherwise C<$TOP/test>
 257 or C<$BLDTOP/test>).
 258
 259 Also, for both C<app> and C<test>, the command may be prefixed with
 260 the content of the environment variable C<$EXE_SHELL>, which is useful
 261 in case OpenSSL has been cross compiled.
 262
 263 =item B<perlapp ARRAYREF, OPTS>
 264
 265 =item B<perltest ARRAYREF, OPTS>
 266
 267 These are also specific applications of C<cmd>, where the interpreter
 268 is predefined to be C<perl>, and they expect the script to be
 269 interpreted to reside in the same location as C<app> and C<test>.
 270
 271 C<perlapp> and C<perltest> will also take the following option:
 272
 273 =over 4
 274
 275 =item B<interpreter_args =E<gt> ARRAYref>
 276
 277 The array reference is a set of arguments for the interpreter rather
 278 than the script.  Take care so that none of them can be seen as a
 279 script!  Flags and their eventual arguments only!
 280
 281 =back
 282
 283 An example:
 284
 285   ok(run(perlapp(["foo.pl", "arg1"],
 286                  interpreter_args => [ "-I", srctop_dir("test") ])));
 287
 288 =back
 289
 290 =begin comment
 291
 292 One might wonder over the complexity of C<apps>, C<fuzz>, C<test>, ...
 293 with all the lazy evaluations and all that.  The reason for this is that
 294 we want to make sure the directory in which those programs are found are
 295 correct at the time these commands are used.  Consider the following code
 296 snippet:
 297
 298   my $cmd = app(["openssl", ...]);
 299
 300   indir "foo", sub {
 301       ok(run($cmd), "Testing foo")
 302   };
 303
 304 If there wasn't this lazy evaluation, the directory where C<openssl> is
 305 found would be incorrect at the time C<run> is called, because it was
 306 calculated before we moved into the directory "foo".
 307
 308 =end comment
 309
 310 =cut
 311
 312 sub cmd {
 313     my $cmd = shift;
 314     my %opts = @_;
 315     return sub {
 316         my $num = shift;
 317         # Make a copy to not destroy the caller's array
 318         my @cmdargs = ( @$cmd );
 319         my @prog = __wrap_cmd(shift @cmdargs, $opts{exe_shell} // ());
 320
 321         return __decorate_cmd($num, [ @prog, quotify(@cmdargs) ],
 322                               %opts);
 323     }
 324 }
 325
 326 sub app {
 327     my $cmd = shift;
 328     my %opts = @_;
 329     return sub {
 330         my @cmdargs = ( @{$cmd} );
 331         my @prog = __fixup_prg(__apps_file(shift @cmdargs, __exeext()));
 332         return cmd([ @prog, @cmdargs ],
 333                    exe_shell => $ENV{EXE_SHELL}, %opts) -> (shift);
 334     }
 335 }
 336
 337 sub fuzz {
 338     my $cmd = shift;
 339     my %opts = @_;
 340     return sub {
 341         my @cmdargs = ( @{$cmd} );
 342         my @prog = __fixup_prg(__fuzz_file(shift @cmdargs, __exeext()));
 343         return cmd([ @prog, @cmdargs ],
 344                    exe_shell => $ENV{EXE_SHELL}, %opts) -> (shift);
 345     }
 346 }
 347
 348 sub test {
 349     my $cmd = shift;
 350     my %opts = @_;
 351     return sub {
 352         my @cmdargs = ( @{$cmd} );
 353         my @prog = __fixup_prg(__test_file(shift @cmdargs, __exeext()));
 354         return cmd([ @prog, @cmdargs ],
 355                    exe_shell => $ENV{EXE_SHELL}, %opts) -> (shift);
 356     }
 357 }
 358
 359 sub perlapp {
 360     my $cmd = shift;
 361     my %opts = @_;
 362     return sub {
 363         my @interpreter_args = defined $opts{interpreter_args} ?
 364             @{$opts{interpreter_args}} : ();
 365         my @interpreter = __fixup_prg($^X);
 366         my @cmdargs = ( @{$cmd} );
 367         my @prog = __apps_file(shift @cmdargs, undef);
 368         return cmd([ @interpreter, @interpreter_args,
 369                      @prog, @cmdargs ], %opts) -> (shift);
 370     }
 371 }
 372
 373 sub perltest {
 374     my $cmd = shift;
 375     my %opts = @_;
 376     return sub {
 377         my @interpreter_args = defined $opts{interpreter_args} ?
 378             @{$opts{interpreter_args}} : ();
 379         my @interpreter = __fixup_prg($^X);
 380         my @cmdargs = ( @{$cmd} );
 381         my @prog = __test_file(shift @cmdargs, undef);
 382         return cmd([ @interpreter, @interpreter_args,
 383                      @prog, @cmdargs ], %opts) -> (shift);
 384     }
 385 }
 386
 387 =over 4
 388
 389 =item B<run CODEREF, OPTS>
 390
 391 CODEREF is expected to be the value return by C<cmd> or any of its
 392 derivatives, anything else will most likely cause an error unless you
 393 know what you're doing.
 394
 395 C<run> executes the command returned by CODEREF and return either the
 396 resulting output (if the option C<capture> is set true) or a boolean
 397 indicating if the command succeeded or not.
 398
 399 The options that C<run> can take are in the form of hash values:
 400
 401 =over 4
 402
 403 =item B<capture =E<gt> 0|1>
 404
 405 If true, the command will be executed with a perl backtick, and C<run> will
 406 return the resulting output as an array of lines.  If false or not given,
 407 the command will be executed with C<system()>, and C<run> will return 1 if
 408 the command was successful or 0 if it wasn't.
 409
 410 =item B<prefix =E<gt> EXPR>
 411
 412 If specified, EXPR will be used as a string to prefix the output from the
 413 command.  This is useful if the output contains lines starting with C<ok >
 414 or C<not ok > that can disturb Test::Harness.
 415
 416 =item B<statusvar =E<gt> VARREF>
 417
 418 If used, B<VARREF> must be a reference to a scalar variable.  It will be
 419 assigned a boolean indicating if the command succeeded or not.  This is
 420 particularly useful together with B<capture>.
 421
 422 =back
 423
 424 For further discussion on what is considered a successful command or not, see
 425 the function C<with> further down.
 426
 427 =back
 428
 429 =cut
 430
 431 sub run {
 432     my ($cmd, $display_cmd) = shift->(0);
 433     my %opts = @_;
 434
 435     return () if !$cmd;
 436
 437     my $prefix = "";
 438     if ( $^O eq "VMS" ) {       # VMS
 439         $prefix = "pipe ";
 440     }
 441
 442     my @r = ();
 443     my $r = 0;
 444     my $e = 0;
 445
 446     die "OpenSSL::Test::run(): statusvar value not a scalar reference"
 447         if $opts{statusvar} && ref($opts{statusvar}) ne "SCALAR";
 448
 449     # In non-verbose, we want to shut up the command interpreter, in case
 450     # it has something to complain about.  On VMS, it might complain both
 451     # on stdout and stderr
 452     my $save_STDOUT;
 453     my $save_STDERR;
 454     if ($ENV{HARNESS_ACTIVE} && !$ENV{HARNESS_VERBOSE}) {
 455         open $save_STDOUT, '>&', \*STDOUT or die "Can't dup STDOUT: $!";
 456         open $save_STDERR, '>&', \*STDERR or die "Can't dup STDERR: $!";
 457         open STDOUT, ">", devnull();
 458         open STDERR, ">", devnull();
 459     }
 460
 461     $ENV{HARNESS_OSSL_LEVEL} = $level + 1;
 462
 463     # The dance we do with $? is the same dance the Unix shells appear to
 464     # do.  For example, a program that gets aborted (and therefore signals
 465     # SIGABRT = 6) will appear to exit with the code 134.  We mimic this
 466     # to make it easier to compare with a manual run of the command.
 467     if ($opts{capture} || defined($opts{prefix})) {
 468         my $pipe;
 469         local $_;
 470
 471         open($pipe, '-|', "$prefix$cmd") or die "Can't start command: $!";
 472         while(<$pipe>) {
 473             my $l = ($opts{prefix} // "") . $_;
 474             if ($opts{capture}) {
 475                 push @r, $l;
 476             } else {
 477                 print STDOUT $l;
 478             }
 479         }
 480         close $pipe;
 481     } else {
 482         $ENV{HARNESS_OSSL_PREFIX} = "# ";
 483         system("$prefix$cmd");
 484         delete $ENV{HARNESS_OSSL_PREFIX};
 485     }
 486     $e = ($? & 0x7f) ? ($? & 0x7f)|0x80 : ($? >> 8);
 487     $r = $hooks{exit_checker}->($e);
 488     if ($opts{statusvar}) {
 489         ${$opts{statusvar}} = $r;
 490     }
 491
 492     if ($ENV{HARNESS_ACTIVE} && !$ENV{HARNESS_VERBOSE}) {
 493         close STDOUT;
 494         close STDERR;
 495         open STDOUT, '>&', $save_STDOUT or die "Can't restore STDOUT: $!";
 496         open STDERR, '>&', $save_STDERR or die "Can't restore STDERR: $!";
 497     }
 498
 499     print STDERR "$prefix$display_cmd => $e\n"
 500         if !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
 501
 502     # At this point, $? stops being interesting, and unfortunately,
 503     # there are Test::More versions that get picky if we leave it
 504     # non-zero.
 505     $? = 0;
 506
 507     if ($opts{capture}) {
 508         return @r;
 509     } else {
 510         return $r;
 511     }
 512 }
 513
 514 END {
 515     my $tb = Test::More->builder;
 516     my $failure = scalar(grep { $_ == 0; } $tb->summary);
 517     if ($failure && $end_with_bailout) {
 518         BAIL_OUT("Stoptest!");
 519     }
 520 }
 521
 522 =head2 Utility functions
 523
 524 The following functions are exported on request when using C<OpenSSL::Test>.
 525
 526   # To only get the bldtop_file and srctop_file functions.
 527   use OpenSSL::Test qw/bldtop_file srctop_file/;
 528
 529   # To only get the bldtop_file function in addition to the default ones.
 530   use OpenSSL::Test qw/:DEFAULT bldtop_file/;
 531
 532 =cut
 533
 534 # Utility functions, exported on request
 535
 536 =over 4
 537
 538 =item B<bldtop_dir LIST>
 539
 540 LIST is a list of directories that make up a path from the top of the OpenSSL
 541 build directory (as indicated by the environment variable C<$TOP> or
 542 C<$BLDTOP>).
 543 C<bldtop_dir> returns the resulting directory as a string, adapted to the local
 544 operating system.
 545
 546 =back
 547
 548 =cut
 549
 550 sub bldtop_dir {
 551     return __bldtop_dir(@_);    # This caters for operating systems that have
 552                                 # a very distinct syntax for directories.
 553 }
 554
 555 =over 4
 556
 557 =item B<bldtop_file LIST, FILENAME>
 558
 559 LIST is a list of directories that make up a path from the top of the OpenSSL
 560 build directory (as indicated by the environment variable C<$TOP> or
 561 C<$BLDTOP>) and FILENAME is the name of a file located in that directory path.
 562 C<bldtop_file> returns the resulting file path as a string, adapted to the local
 563 operating system.
 564
 565 =back
 566
 567 =cut
 568
 569 sub bldtop_file {
 570     return __bldtop_file(@_);
 571 }
 572
 573 =over 4
 574
 575 =item B<srctop_dir LIST>
 576
 577 LIST is a list of directories that make up a path from the top of the OpenSSL
 578 source directory (as indicated by the environment variable C<$TOP> or
 579 C<$SRCTOP>).
 580 C<srctop_dir> returns the resulting directory as a string, adapted to the local
 581 operating system.
 582
 583 =back
 584
 585 =cut
 586
 587 sub srctop_dir {
 588     return __srctop_dir(@_);    # This caters for operating systems that have
 589                                 # a very distinct syntax for directories.
 590 }
 591
 592 =over 4
 593
 594 =item B<srctop_file LIST, FILENAME>
 595
 596 LIST is a list of directories that make up a path from the top of the OpenSSL
 597 source directory (as indicated by the environment variable C<$TOP> or
 598 C<$SRCTOP>) and FILENAME is the name of a file located in that directory path.
 599 C<srctop_file> returns the resulting file path as a string, adapted to the local
 600 operating system.
 601
 602 =back
 603
 604 =cut
 605
 606 sub srctop_file {
 607     return __srctop_file(@_);
 608 }
 609
 610 =over 4
 611
 612 =item B<data_dir LIST>
 613
 614 LIST is a list of directories that make up a path from the data directory
 615 associated with the test (see L</DESCRIPTION> above).
 616 C<data_dir> returns the resulting directory as a string, adapted to the local
 617 operating system.
 618
 619 =back
 620
 621 =cut
 622
 623 sub data_dir {
 624     return __data_dir(@_);
 625 }
 626
 627 =over 4
 628
 629 =item B<data_file LIST, FILENAME>
 630
 631 LIST is a list of directories that make up a path from the data directory
 632 associated with the test (see L</DESCRIPTION> above) and FILENAME is the name
 633 of a file located in that directory path.  C<data_file> returns the resulting
 634 file path as a string, adapted to the local operating system.
 635
 636 =back
 637
 638 =cut
 639
 640 sub data_file {
 641     return __data_file(@_);
 642 }
 643
 644 =over 4
 645
 646 =item B<pipe LIST>
 647
 648 LIST is a list of CODEREFs returned by C<app> or C<test>, from which C<pipe>
 649 creates a new command composed of all the given commands put together in a
 650 pipe.  C<pipe> returns a new CODEREF in the same manner as C<app> or C<test>,
 651 to be passed to C<run> for execution.
 652
 653 =back
 654
 655 =cut
 656
 657 sub pipe {
 658     my @cmds = @_;
 659     return
 660         sub {
 661             my @cs  = ();
 662             my @dcs = ();
 663             my @els = ();
 664             my $counter = 0;
 665             foreach (@cmds) {
 666                 my ($c, $dc, @el) = $_->(++$counter);
 667
 668                 return () if !$c;
 669
 670                 push @cs, $c;
 671                 push @dcs, $dc;
 672                 push @els, @el;
 673             }
 674             return (
 675                 join(" | ", @cs),
 676                 join(" | ", @dcs),
 677                 @els
 678                 );
 679     };
 680 }
 681
 682 =over 4
 683
 684 =item B<with HASHREF, CODEREF>
 685
 686 C<with> will temporarily install hooks given by the HASHREF and then execute
 687 the given CODEREF.  Hooks are usually expected to have a coderef as value.
 688
 689 The currently available hoosk are:
 690
 691 =over 4
 692
 693 =item B<exit_checker =E<gt> CODEREF>
 694
 695 This hook is executed after C<run> has performed its given command.  The
 696 CODEREF receives the exit code as only argument and is expected to return
 697 1 (if the exit code indicated success) or 0 (if the exit code indicated
 698 failure).
 699
 700 =back
 701
 702 =back
 703
 704 =cut
 705
 706 sub with {
 707     my $opts = shift;
 708     my %opts = %{$opts};
 709     my $codeblock = shift;
 710
 711     my %saved_hooks = ();
 712
 713     foreach (keys %opts) {
 714         $saved_hooks{$_} = $hooks{$_}   if exists($hooks{$_});
 715         $hooks{$_} = $opts{$_};
 716     }
 717
 718     $codeblock->();
 719
 720     foreach (keys %saved_hooks) {
 721         $hooks{$_} = $saved_hooks{$_};
 722     }
 723 }
 724
 725 =over 4
 726
 727 =item B<cmdstr CODEREF, OPTS>
 728
 729 C<cmdstr> takes a CODEREF from C<app> or C<test> and simply returns the
 730 command as a string.
 731
 732 C<cmdstr> takes some additional options OPTS that affect the string returned:
 733
 734 =over 4
 735
 736 =item B<display =E<gt> 0|1>
 737
 738 When set to 0, the returned string will be with all decorations, such as a
 739 possible redirect of stderr to the null device.  This is suitable if the
 740 string is to be used directly in a recipe.
 741
 742 When set to 1, the returned string will be without extra decorations.  This
 743 is suitable for display if that is desired (doesn't confuse people with all
 744 internal stuff), or if it's used to pass a command down to a subprocess.
 745
 746 Default: 0
 747
 748 =back
 749
 750 =back
 751
 752 =cut
 753
 754 sub cmdstr {
 755     my ($cmd, $display_cmd) = shift->(0);
 756     my %opts = @_;
 757
 758     if ($opts{display}) {
 759         return $display_cmd;
 760     } else {
 761         return $cmd;
 762     }
 763 }
 764
 765 =over 4
 766
 767 =item B<quotify LIST>
 768
 769 LIST is a list of strings that are going to be used as arguments for a
 770 command, and makes sure to inject quotes and escapes as necessary depending
 771 on the content of each string.
 772
 773 This can also be used to put quotes around the executable of a command.
 774 I<This must never ever be done on VMS.>
 775
 776 =back
 777
 778 =cut
 779
 780 sub quotify {
 781     # Unix setup (default if nothing else is mentioned)
 782     my $arg_formatter =
 783         sub { $_ = shift;
 784               ($_ eq '' || /\s|[\{\}\\\$\[\]\*\?\|\&:;<>]/) ? "'$_'" : $_ };
 785
 786     if ( $^O eq "VMS") {        # VMS setup
 787         $arg_formatter = sub {
 788             $_ = shift;
 789             if ($_ eq '' || /\s|["[:upper:]]/) {
 790                 s/"/""/g;
 791                 '"'.$_.'"';
 792             } else {
 793                 $_;
 794             }
 795         };
 796     } elsif ( $^O eq "MSWin32") { # MSWin setup
 797         $arg_formatter = sub {
 798             $_ = shift;
 799             if ($_ eq '' || /\s|["\|\&\*\;<>]/) {
 800                 s/(["\\])/\\$1/g;
 801                 '"'.$_.'"';
 802             } else {
 803                 $_;
 804             }
 805         };
 806     }
 807
 808     return map { $arg_formatter->($_) } @_;
 809 }
 810
 811 =over 4
 812
 813 =item B<openssl_versions>
 814
 815 Returns a list of two version numbers, the first representing the build
 816 version, the second representing the library version.  See opensslv.h for
 817 more information on those numbers.
 818
 819 =back
 820
 821 =cut
 822
 823 my @versions = ();
 824 sub openssl_versions {
 825     unless (@versions) {
 826         my %lines =
 827             map { s/\R$//;
 828                   /^(.*): (.*)$/;
 829                   $1 => $2 }
 830             run(test(['versions']), capture => 1);
 831         @versions = ( $lines{'Build version'}, $lines{'Library version'} );
 832     }
 833     return @versions;
 834 }
 835
 836 =over 4
 837
 838 =item B<ok_nofips EXPR, TEST_NAME>
 839
 840 C<ok_nofips> is equivalent to using C<ok> when the environment variable
 841 C<FIPS_MODE> is undefined, otherwise it is equivalent to C<not ok>. This can be
 842 used for C<ok> tests that must fail when testing a FIPS provider. The parameters
 843 are the same as used by C<ok> which is an expression EXPR followed by the test
 844 description TEST_NAME.
 845
 846 An example:
 847
 848   ok_nofips(run(app(["md5.pl"])), "md5 should fail in fips mode");
 849
 850 =item B<is_nofips EXPR1, EXPR2, TEST_NAME>
 851
 852 C<is_nofips> is equivalent to using C<is> when the environment variable
 853 C<FIPS_MODE> is undefined, otherwise it is equivalent to C<isnt>. This can be
 854 used for C<is> tests that must fail when testing a FIPS provider. The parameters
 855 are the same as used by C<is> which has 2 arguments EXPR1 and EXPR2 that can be
 856 compared using eq or ne, followed by a test description TEST_NAME.
 857
 858 An example:
 859
 860   is_nofips(ultimate_answer(), 42,  "Meaning of Life");
 861
 862 =item B<isnt_nofips EXPR1, EXPR2, TEST_NAME>
 863
 864 C<isnt_nofips> is equivalent to using C<isnt> when the environment variable
 865 C<FIPS_MODE> is undefined, otherwise it is equivalent to C<is>. This can be
 866 used for C<isnt> tests that must fail when testing a FIPS provider. The
 867 parameters are the same as used by C<isnt> which has 2 arguments EXPR1 and EXPR2
 868 that can be compared using ne or eq, followed by a test description TEST_NAME.
 869
 870 An example:
 871
 872   isnt_nofips($foo, '',  "Got some foo");
 873
 874 =back
 875
 876 =cut
 877
 878 sub ok_nofips {
 879     return ok(!$_[0], @_[1..$#_]) if defined $ENV{FIPS_MODE};
 880     return ok($_[0], @_[1..$#_]);
 881 }
 882
 883 sub is_nofips {
 884     return isnt($_[0], $_[1], @_[2..$#_]) if defined $ENV{FIPS_MODE};
 885     return is($_[0], $_[1], @_[2..$#_]);
 886 }
 887
 888 sub isnt_nofips {
 889     return is($_[0], $_[1], @_[2..$#_]) if defined $ENV{FIPS_MODE};
 890     return isnt($_[0], $_[1], @_[2..$#_]);
 891 }
 892
 893 ######################################################################
 894 # private functions.  These are never exported.
 895
 896 =head1 ENVIRONMENT
 897
 898 OpenSSL::Test depends on some environment variables.
 899
 900 =over 4
 901
 902 =item B<TOP>
 903
 904 This environment variable is mandatory.  C<setup> will check that it's
 905 defined and that it's a directory that contains the file C<Configure>.
 906 If this isn't so, C<setup> will C<BAIL_OUT>.
 907
 908 =item B<BIN_D>
 909
 910 If defined, its value should be the directory where the openssl application
 911 is located.  Defaults to C<$TOP/apps> (adapted to the operating system).
 912
 913 =item B<TEST_D>
 914
 915 If defined, its value should be the directory where the test applications
 916 are located.  Defaults to C<$TOP/test> (adapted to the operating system).
 917
 918 =item B<STOPTEST>
 919
 920 If defined, it puts testing in a different mode, where a recipe with
 921 failures will result in a C<BAIL_OUT> at the end of its run.
 922
 923 =item B<FIPS_MODE>
 924
 925 If defined it indicates that the FIPS provider is being tested. Tests may use
 926 B<ok_nofips>, B<is_nofips> and B<isnt_nofips> to invert test results
 927 i.e. Some tests may only work in non FIPS mode.
 928
 929 =back
 930
 931 =cut
 932
 933 sub __env {
 934     (my $recipe_datadir = basename($0)) =~ s/\.t$/_data/i;
 935
 936     $directories{SRCTOP}  = abs_path($ENV{SRCTOP} || $ENV{TOP});
 937     $directories{BLDTOP}  = abs_path($ENV{BLDTOP} || $ENV{TOP});
 938     $directories{BLDAPPS} = $ENV{BIN_D}  || __bldtop_dir("apps");
 939     $directories{SRCAPPS} =                 __srctop_dir("apps");
 940     $directories{BLDFUZZ} =                 __bldtop_dir("fuzz");
 941     $directories{SRCFUZZ} =                 __srctop_dir("fuzz");
 942     $directories{BLDTEST} = $ENV{TEST_D} || __bldtop_dir("test");
 943     $directories{SRCTEST} =                 __srctop_dir("test");
 944     $directories{SRCDATA} =                 __srctop_dir("test", "recipes",
 945                                                          $recipe_datadir);
 946     $directories{RESULTS} = $ENV{RESULT_D} || $directories{BLDTEST};
 947
 948     push @direnv, "TOP"       if $ENV{TOP};
 949     push @direnv, "SRCTOP"    if $ENV{SRCTOP};
 950     push @direnv, "BLDTOP"    if $ENV{BLDTOP};
 951     push @direnv, "BIN_D"     if $ENV{BIN_D};
 952     push @direnv, "TEST_D"    if $ENV{TEST_D};
 953     push @direnv, "RESULT_D"  if $ENV{RESULT_D};
 954
 955     $end_with_bailout     = $ENV{STOPTEST} ? 1 : 0;
 956 };
 957
 958 # __srctop_file and __srctop_dir are helpers to build file and directory
 959 # names on top of the source directory.  They depend on $SRCTOP, and
 960 # therefore on the proper use of setup() and when needed, indir().
 961 # __bldtop_file and __bldtop_dir do the same thing but relative to $BLDTOP.
 962 # __srctop_file and __bldtop_file take the same kind of argument as
 963 # File::Spec::Functions::catfile.
 964 # Similarly, __srctop_dir and __bldtop_dir take the same kind of argument
 965 # as File::Spec::Functions::catdir
 966 sub __srctop_file {
 967     BAIL_OUT("Must run setup() first") if (! $test_name);
 968
 969     my $f = pop;
 970     return catfile($directories{SRCTOP},@_,$f);
 971 }
 972
 973 sub __srctop_dir {
 974     BAIL_OUT("Must run setup() first") if (! $test_name);
 975
 976     return catdir($directories{SRCTOP},@_);
 977 }
 978
 979 sub __bldtop_file {
 980     BAIL_OUT("Must run setup() first") if (! $test_name);
 981
 982     my $f = pop;
 983     return catfile($directories{BLDTOP},@_,$f);
 984 }
 985
 986 sub __bldtop_dir {
 987     BAIL_OUT("Must run setup() first") if (! $test_name);
 988
 989     return catdir($directories{BLDTOP},@_);
 990 }
 991
 992 # __exeext is a function that returns the platform dependent file extension
 993 # for executable binaries, or the value of the environment variable $EXE_EXT
 994 # if that one is defined.
 995 sub __exeext {
 996     my $ext = "";
 997     if ($^O eq "VMS" ) {        # VMS
 998         $ext = ".exe";
 999     } elsif ($^O eq "MSWin32") { # Windows
1000         $ext = ".exe";
1001     }
1002     return $ENV{"EXE_EXT"} || $ext;
1003 }
1004
1005 # __test_file, __apps_file and __fuzz_file return the full path to a file
1006 # relative to the test/, apps/ or fuzz/ directory in the build tree or the
1007 # source tree, depending on where the file is found.  Note that when looking
1008 # in the build tree, the file name with an added extension is looked for, if
1009 # an extension is given.  The intent is to look for executable binaries (in
1010 # the build tree) or possibly scripts (in the source tree).
1011 # These functions all take the same arguments as File::Spec::Functions::catfile,
1012 # *plus* a mandatory extension argument.  This extension argument can be undef,
1013 # and is ignored in such a case.
1014 sub __test_file {
1015     BAIL_OUT("Must run setup() first") if (! $test_name);
1016
1017     my $e = pop || "";
1018     my $f = pop;
1019     my $out = catfile($directories{BLDTEST},@_,$f . $e);
1020     $out = catfile($directories{SRCTEST},@_,$f) unless -f $out;
1021     return $out;
1022 }
1023
1024 sub __apps_file {
1025     BAIL_OUT("Must run setup() first") if (! $test_name);
1026
1027     my $e = pop || "";
1028     my $f = pop;
1029     my $out = catfile($directories{BLDAPPS},@_,$f . $e);
1030     $out = catfile($directories{SRCAPPS},@_,$f) unless -f $out;
1031     return $out;
1032 }
1033
1034 sub __fuzz_file {
1035     BAIL_OUT("Must run setup() first") if (! $test_name);
1036
1037     my $e = pop || "";
1038     my $f = pop;
1039     my $out = catfile($directories{BLDFUZZ},@_,$f . $e);
1040     $out = catfile($directories{SRCFUZZ},@_,$f) unless -f $out;
1041     return $out;
1042 }
1043
1044 sub __data_file {
1045     BAIL_OUT("Must run setup() first") if (! $test_name);
1046
1047     my $f = pop;
1048     return catfile($directories{SRCDATA},@_,$f);
1049 }
1050
1051 sub __data_dir {
1052     BAIL_OUT("Must run setup() first") if (! $test_name);
1053
1054     return catdir($directories{SRCDATA},@_);
1055 }
1056
1057 sub __results_file {
1058     BAIL_OUT("Must run setup() first") if (! $test_name);
1059
1060     my $f = pop;
1061     return catfile($directories{RESULTS},@_,$f);
1062 }
1063
1064 # __cwd DIR
1065 # __cwd DIR, OPTS
1066 #
1067 # __cwd changes directory to DIR (string) and changes all the relative
1068 # entries in %directories accordingly.  OPTS is an optional series of
1069 # hash style arguments to alter __cwd's behavior:
1070 #
1071 #    create = 0|1       The directory we move to is created if 1, not if 0.
1072 #    cleanup = 0|1      The directory we move from is removed if 1, not if 0.
1073
1074 sub __cwd {
1075     my $dir = catdir(shift);
1076     my %opts = @_;
1077     my $abscurdir = rel2abs(curdir());
1078     my $absdir = rel2abs($dir);
1079     my $reverse = abs2rel($abscurdir, $absdir);
1080
1081     # PARANOIA: if we're not moving anywhere, we do nothing more
1082     if ($abscurdir eq $absdir) {
1083         return $reverse;
1084     }
1085
1086     # Do not support a move to a different volume for now.  Maybe later.
1087     BAIL_OUT("FAILURE: \"$dir\" moves to a different volume, not supported")
1088         if $reverse eq $abscurdir;
1089
1090     # If someone happened to give a directory that leads back to the current,
1091     # it's extremely silly to do anything more, so just simulate that we did
1092     # move.
1093     # In this case, we won't even clean it out, for safety's sake.
1094     return "." if $reverse eq "";
1095
1096     $dir = canonpath($dir);
1097     if ($opts{create}) {
1098         mkpath($dir);
1099     }
1100
1101     # We are recalculating the directories we keep track of, but need to save
1102     # away the result for after having moved into the new directory.
1103     my %tmp_directories = ();
1104     my %tmp_ENV = ();
1105
1106     # For each of these directory variables, figure out where they are relative
1107     # to the directory we want to move to if they aren't absolute (if they are,
1108     # they don't change!)
1109     my @dirtags = sort keys %directories;
1110     foreach (@dirtags) {
1111         if (!file_name_is_absolute($directories{$_})) {
1112             my $newpath = abs2rel(rel2abs($directories{$_}), rel2abs($dir));
1113             $tmp_directories{$_} = $newpath;
1114         }
1115     }
1116
1117     # Treat each environment variable that was used to get us the values in
1118     # %directories the same was as the paths in %directories, so any sub
1119     # process can use their values properly as well
1120     foreach (@direnv) {
1121         if (!file_name_is_absolute($ENV{$_})) {
1122             my $newpath = abs2rel(rel2abs($ENV{$_}), rel2abs($dir));
1123             $tmp_ENV{$_} = $newpath;
1124         }
1125     }
1126
1127     # Should we just bail out here as well?  I'm unsure.
1128     return undef unless chdir($dir);
1129
1130     if ($opts{cleanup}) {
1131         rmtree(".", { safe => 0, keep_root => 1 });
1132     }
1133
1134     # We put back new values carefully.  Doing the obvious
1135     # %directories = ( %tmp_directories )
1136     # will clear out any value that happens to be an absolute path
1137     foreach (keys %tmp_directories) {
1138         $directories{$_} = $tmp_directories{$_};
1139     }
1140     foreach (keys %tmp_ENV) {
1141         $ENV{$_} = $tmp_ENV{$_};
1142     }
1143
1144     if ($debug) {
1145         print STDERR "DEBUG: __cwd(), directories and files:\n";
1146         print STDERR "  \$directories{BLDTEST} = \"$directories{BLDTEST}\"\n";
1147         print STDERR "  \$directories{SRCTEST} = \"$directories{SRCTEST}\"\n";
1148         print STDERR "  \$directories{SRCDATA} = \"$directories{SRCDATA}\"\n";
1149         print STDERR "  \$directories{RESULTS} = \"$directories{RESULTS}\"\n";
1150         print STDERR "  \$directories{BLDAPPS} = \"$directories{BLDAPPS}\"\n";
1151         print STDERR "  \$directories{SRCAPPS} = \"$directories{SRCAPPS}\"\n";
1152         print STDERR "  \$directories{SRCTOP}  = \"$directories{SRCTOP}\"\n";
1153         print STDERR "  \$directories{BLDTOP}  = \"$directories{BLDTOP}\"\n";
1154         print STDERR "\n";
1155         print STDERR "  current directory is \"",curdir(),"\"\n";
1156         print STDERR "  the way back is \"$reverse\"\n";
1157     }
1158
1159     return $reverse;
1160 }
1161
1162 # __wrap_cmd CMD
1163 # __wrap_cmd CMD, EXE_SHELL
1164 #
1165 # __wrap_cmd "wraps" CMD (string) with a beginning command that makes sure
1166 # the command gets executed with an appropriate environment.  If EXE_SHELL
1167 # is given, it is used as the beginning command.
1168 #
1169 # __wrap_cmd returns a list that should be used to build up a larger list
1170 # of command tokens, or be joined together like this:
1171 #
1172 #    join(" ", __wrap_cmd($cmd))
1173 sub __wrap_cmd {
1174     my $cmd = shift;
1175     my $exe_shell = shift;
1176
1177     my @prefix = ( __bldtop_file("util", "shlib_wrap.sh") );
1178
1179     if(defined($exe_shell)) {
1180         @prefix = ( $exe_shell );
1181     } elsif ($^O eq "VMS" || $^O eq "MSWin32") {
1182         # VMS and Windows don't use any wrapper script for the moment
1183         @prefix = ();
1184     }
1185
1186     return (@prefix, $cmd);
1187 }
1188
1189 # __fixup_prg PROG
1190 #
1191 # __fixup_prg does whatever fixup is needed to execute an executable binary
1192 # given by PROG (string).
1193 #
1194 # __fixup_prg returns a string with the possibly prefixed program path spec.
1195 sub __fixup_prg {
1196     my $prog = shift;
1197
1198     my $prefix = "";
1199
1200     if ($^O eq "VMS" ) {
1201         $prefix = ($prog =~ /^(?:[\$a-z0-9_]+:)?[<\[]/i ? "mcr " : "mcr []");
1202     }
1203
1204     if (defined($prog)) {
1205         # Make sure to quotify the program file on platforms that may
1206         # have spaces or similar in their path name.
1207         # To our knowledge, VMS is the exception where quotifying should
1208         # never happen.
1209         ($prog) = quotify($prog) unless $^O eq "VMS";
1210         return $prefix.$prog;
1211     }
1212
1213     print STDERR "$prog not found\n";
1214     return undef;
1215 }
1216
1217 # __decorate_cmd NUM, CMDARRAYREF
1218 #
1219 # __decorate_cmd takes a command number NUM and a command token array
1220 # CMDARRAYREF, builds up a command string from them and decorates it
1221 # with necessary redirections.
1222 # __decorate_cmd returns a list of two strings, one with the command
1223 # string to actually be used, the other to be displayed for the user.
1224 # The reason these strings might differ is that we redirect stderr to
1225 # the null device unless we're verbose and unless the user has
1226 # explicitly specified a stderr redirection.
1227 sub __decorate_cmd {
1228     BAIL_OUT("Must run setup() first") if (! $test_name);
1229
1230     my $num = shift;
1231     my $cmd = shift;
1232     my %opts = @_;
1233
1234     my $cmdstr = join(" ", @$cmd);
1235     my $null = devnull();
1236     my $fileornull = sub { $_[0] ? $_[0] : $null; };
1237     my $stdin = "";
1238     my $stdout = "";
1239     my $stderr = "";
1240     my $saved_stderr = undef;
1241     $stdin = " < ".$fileornull->($opts{stdin})  if exists($opts{stdin});
1242     $stdout= " > ".$fileornull->($opts{stdout}) if exists($opts{stdout});
1243     $stderr=" 2> ".$fileornull->($opts{stderr}) if exists($opts{stderr});
1244
1245     my $display_cmd = "$cmdstr$stdin$stdout$stderr";
1246
1247     $stderr=" 2> ".$null
1248         unless $stderr || !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
1249
1250     $cmdstr .= "$stdin$stdout$stderr";
1251
1252     if ($debug) {
1253         print STDERR "DEBUG[__decorate_cmd]: \$cmdstr = \"$cmdstr\"\n";
1254         print STDERR "DEBUG[__decorate_cmd]: \$display_cmd = \"$display_cmd\"\n";
1255     }
1256
1257     return ($cmdstr, $display_cmd);
1258 }
1259
1260 =head1 SEE ALSO
1261
1262 L<Test::More>, L<Test::Harness>
1263
1264 =head1 AUTHORS
1265
1266 Richard Levitte E<lt>levitte@openssl.orgE<gt> with assistance and
1267 inspiration from Andy Polyakov E<lt>appro@openssl.org<gt>.
1268
1269 =cut
1270
1271 no warnings 'redefine';
1272 sub subtest {
1273     $level++;
1274
1275     Test::More::subtest @_;
1276
1277     $level--;
1278 };
1279
1280 1;