test/run-stp-tests.sh

   1 #!/bin/sh
   2 #
   3 # "$Id: run-stp-tests.sh 6649 2007-07-11 21:46:42Z mike $"
   4 #
   5 #   Perform the complete set of IPP compliance tests specified in the
   6 #   CUPS Software Test Plan.
   7 #
   8 #   Copyright 2007 by Apple Inc.
   9 #   Copyright 1997-2007 by Easy Software Products, all rights reserved.
  10 #
  11 #   These coded instructions, statements, and computer programs are the
  12 #   property of Apple Inc. and are protected by Federal copyright
  13 #   law.  Distribution and use rights are outlined in the file "LICENSE.txt"
  14 #   which should have been included with this file.  If this file is
  15 #   file is missing or damaged, see the license at "http://www.cups.org/".
  16 #
  17
  18 argcount=$#
  19
  20 #
  21 # Make the IPP test program...
  22 #
  23
  24 make
  25
  26 #
  27 # Figure out the proper echo options...
  28 #
  29
  30 if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null; then
  31         ac_n=-n
  32         ac_c=
  33 else
  34         ac_n=
  35         ac_c='\c'
  36 fi
  37
  38 #
  39 # Greet the tester...
  40 #
  41
  42 echo "Welcome to the CUPS Automated Test Script."
  43 echo ""
  44 echo "Before we begin, it is important that you understand that the larger"
  45 echo "tests require significant amounts of RAM and disk space.  If you"
  46 echo "attempt to run one of the big tests on a system that lacks sufficient"
  47 echo "disk and virtual memory, the UNIX kernel might decide to kill one or"
  48 echo "more system processes that you've grown attached to, like the X"
  49 echo "server.  The question you may want to ask yourself before running a"
  50 echo "large test is: Do you feel lucky?"
  51 echo ""
  52 echo "OK, now that we have the Dirty Harry quote out of the way, please"
  53 echo "choose the type of test you wish to perform:"
  54 echo ""
  55 echo "0 - No testing, keep the scheduler running for me (all systems)"
  56 echo "1 - Basic conformance test, no load testing (all systems)"
  57 echo "2 - Basic conformance test, some load testing (minimum 256MB VM, 50MB disk)"
  58 echo "3 - Basic conformance test, extreme load testing (minimum 1GB VM, 500MB disk)"
  59 echo "4 - Basic conformance test, torture load testing (minimum 2GB VM, 1GB disk)"
  60 echo ""
  61 echo $ac_n "Enter the number of the test you wish to perform: [1] $ac_c"
  62
  63 if test $# -gt 0; then
  64         testtype=$1
  65         shift
  66 else
  67         read testtype
  68 fi
  69 echo ""
  70
  71 case "$testtype" in
  72         0)
  73                 echo "Running in test mode (0)"
  74                 nprinters1=0
  75                 nprinters2=0
  76                 pjobs=0
  77                 ;;
  78         2)
  79                 echo "Running the medium tests (2)"
  80                 nprinters1=10
  81                 nprinters2=20
  82                 pjobs=20
  83                 ;;
  84         3)
  85                 echo "Running the extreme tests (3)"
  86                 nprinters1=500
  87                 nprinters2=1000
  88                 pjobs=100
  89                 ;;
  90         4)
  91                 echo "Running the torture tests (4)"
  92                 nprinters1=10000
  93                 nprinters2=20000
  94                 pjobs=200
  95                 ;;
  96         *)
  97                 echo "Running the timid tests (1)"
  98                 nprinters1=0
  99                 nprinters2=0
 100                 pjobs=0
 101                 ;;
 102 esac
 103
 104 #
 105 # See if we want to do SSL testing...
 106 #
 107
 108 echo ""
 109 echo "Now you can choose whether to create a SSL/TLS encryption key and"
 110 echo "certificate for testing; these tests currently require the OpenSSL"
 111 echo "tools:"
 112 echo ""
 113 echo "0 - Do not do SSL/TLS encryption tests"
 114 echo "1 - Test but do not require encryption"
 115 echo "2 - Test and require encryption"
 116 echo ""
 117 echo $ac_n "Enter the number of the SSL/TLS tests to perform: [0] $ac_c"
 118
 119 if test $# -gt 0; then
 120         ssltype=$1
 121         shift
 122 else
 123         read ssltype
 124 fi
 125 echo ""
 126
 127 case "$ssltype" in
 128         1)
 129                 echo "Will test but not require encryption (1)"
 130                 ;;
 131         2)
 132                 echo "Will test and require encryption (2)"
 133                 ;;
 134         *)
 135                 echo "Not using SSL/TLS (0)"
 136                 ssltype=0
 137                 ;;
 138 esac
 139
 140 #
 141 # Information for the server/tests...
 142 #
 143
 144 user="$USER"
 145 if test -z "$user"; then
 146         if test -x /usr/ucb/whoami; then
 147                 user=`/usr/ucb/whoami`
 148         else
 149                 user=`whoami`
 150         fi
 151
 152         if test -z "$user"; then
 153                 user="unknown"
 154         fi
 155 fi
 156
 157 port=8631
 158 cwd=`pwd`
 159 root=`dirname $cwd`
 160
 161 #
 162 # Make sure that the LPDEST and PRINTER environment variables are
 163 # not included in the environment that is passed to the tests.  These
 164 # will usually cause tests to fail erroneously...
 165 #
 166
 167 typeset +x LPDEST
 168 typeset +x PRINTER
 169
 170 #
 171 # See if we want to use valgrind...
 172 #
 173
 174 echo ""
 175 echo "This test script can use the Valgrind software from:"
 176 echo ""
 177 echo "    http://developer.kde.org/~sewardj/"
 178 echo ""
 179 echo $ac_n "Enter Y to use Valgrind or N to not use Valgrind: [N] $ac_c"
 180
 181 if test $# -gt 0; then
 182         usevalgrind=$1
 183         shift
 184 else
 185         read usevalgrind
 186 fi
 187 echo ""
 188
 189 case "$usevalgrind" in
 190         Y* | y*)
 191                 valgrind="valgrind --tool=memcheck --log-file=/tmp/cups-$user/log/valgrind --error-limit=no --leak-check=yes --trace-children=yes"
 192                 echo "Using Valgrind; log files can be found in /tmp/cups-$user/log..."
 193                 ;;
 194
 195         *)
 196                 valgrind=""
 197                 ;;
 198 esac
 199
 200 #
 201 # Start by creating temporary directories for the tests...
 202 #
 203
 204 echo "Creating directories for test..."
 205
 206 rm -rf /tmp/cups-$user
 207 mkdir /tmp/cups-$user
 208 mkdir /tmp/cups-$user/bin
 209 mkdir /tmp/cups-$user/bin/backend
 210 mkdir /tmp/cups-$user/bin/filter
 211 mkdir /tmp/cups-$user/certs
 212 mkdir /tmp/cups-$user/share
 213 mkdir /tmp/cups-$user/share/banners
 214 mkdir /tmp/cups-$user/share/model
 215 mkdir /tmp/cups-$user/interfaces
 216 mkdir /tmp/cups-$user/log
 217 mkdir /tmp/cups-$user/ppd
 218 mkdir /tmp/cups-$user/spool
 219 mkdir /tmp/cups-$user/spool/temp
 220 mkdir /tmp/cups-$user/ssl
 221
 222 ln -s $root/backend/http /tmp/cups-$user/bin/backend
 223 ln -s $root/backend/ipp /tmp/cups-$user/bin/backend
 224 ln -s $root/backend/lpd /tmp/cups-$user/bin/backend
 225 ln -s $root/backend/parallel /tmp/cups-$user/bin/backend
 226 ln -s $root/backend/serial /tmp/cups-$user/bin/backend
 227 ln -s $root/backend/snmp /tmp/cups-$user/bin/backend
 228 ln -s $root/backend/socket /tmp/cups-$user/bin/backend
 229 ln -s $root/backend/usb /tmp/cups-$user/bin/backend
 230 ln -s $root/cgi-bin /tmp/cups-$user/bin
 231 ln -s $root/monitor /tmp/cups-$user/bin
 232 ln -s $root/notifier /tmp/cups-$user/bin
 233 ln -s $root/scheduler /tmp/cups-$user/bin/daemon
 234 ln -s $root/filter/hpgltops /tmp/cups-$user/bin/filter
 235 ln -s $root/filter/imagetops /tmp/cups-$user/bin/filter
 236 ln -s $root/filter/imagetoraster /tmp/cups-$user/bin/filter
 237 ln -s $root/filter/pstops /tmp/cups-$user/bin/filter
 238 ln -s $root/filter/rastertoepson /tmp/cups-$user/bin/filter
 239 ln -s $root/filter/rastertohp /tmp/cups-$user/bin/filter
 240 ln -s $root/filter/texttops /tmp/cups-$user/bin/filter
 241 ln -s $root/pdftops/pdftops /tmp/cups-$user/bin/filter
 242
 243 ln -s $root/data/classified /tmp/cups-$user/share/banners
 244 ln -s $root/data/confidential /tmp/cups-$user/share/banners
 245 ln -s $root/data/secret /tmp/cups-$user/share/banners
 246 ln -s $root/data/standard /tmp/cups-$user/share/banners
 247 ln -s $root/data/topsecret /tmp/cups-$user/share/banners
 248 ln -s $root/data/unclassified /tmp/cups-$user/share/banners
 249 ln -s $root/data /tmp/cups-$user/share/charmaps
 250 ln -s $root/data /tmp/cups-$user/share/charsets
 251 ln -s $root/data /tmp/cups-$user/share
 252 ln -s $root/fonts /tmp/cups-$user/share
 253 ln -s $root/ppd/*.ppd /tmp/cups-$user/share/model
 254 ln -s $root/templates /tmp/cups-$user/share
 255
 256 if test $ssltype != 0; then
 257         mkdir $root/ssl
 258         cp server.* $root/ssl
 259 fi
 260
 261 #
 262 # Mac OS X filters and configuration files...
 263 #
 264
 265 if test `uname` = Darwin; then
 266         ln -s /usr/libexec/cups/filter/cgpdfto* /tmp/cups-$user/bin/filter
 267         ln -s /usr/libexec/cups/filter/nsimagetopdf /tmp/cups-$user/bin/filter
 268         ln -s /usr/libexec/cups/filter/nstexttopdf /tmp/cups-$user/bin/filter
 269         ln -s /usr/libexec/cups/filter/pictwpstops /tmp/cups-$user/bin/filter
 270         ln -s /usr/libexec/cups/filter/pstoappleps /tmp/cups-$user/bin/filter
 271         ln -s /usr/libexec/cups/filter/pstocupsraster /tmp/cups-$user/bin/filter
 272         ln -s /usr/libexec/cups/filter/pstopdffilter /tmp/cups-$user/bin/filter
 273
 274         ln -s /private/etc/cups/apple.* /tmp/cups-$user
 275 fi
 276
 277
 278 #
 279 # Then create the necessary config files...
 280 #
 281
 282 echo "Creating cupsd.conf for test..."
 283
 284 if test $ssltype = 2; then
 285         encryption="Encryption Required"
 286 else
 287         encryption=""
 288 fi
 289
 290 cat >/tmp/cups-$user/cupsd.conf <<EOF
 291 Browsing Off
 292 FileDevice yes
 293 Printcap
 294 Listen 127.0.0.1:$port
 295 User $user
 296 ServerRoot /tmp/cups-$user
 297 StateDir /tmp/cups-$user
 298 ServerBin /tmp/cups-$user/bin
 299 CacheDir /tmp/cups-$user/share
 300 DataDir /tmp/cups-$user/share
 301 FontPath /tmp/cups-$user/share/fonts
 302 DocumentRoot $root/doc
 303 RequestRoot /tmp/cups-$user/spool
 304 TempDir /tmp/cups-$user/spool/temp
 305 MaxLogSize 0
 306 AccessLog /tmp/cups-$user/log/access_log
 307 ErrorLog /tmp/cups-$user/log/error_log
 308 PageLog /tmp/cups-$user/log/page_log
 309 LogLevel debug
 310 PreserveJobHistory Yes
 311 <Policy default>
 312 <Limit All>
 313 Order Deny,Allow
 314 Deny from all
 315 Allow from 127.0.0.1
 316 $encryption
 317 </Limit>
 318 </Policy>
 319 EOF
 320
 321 touch /tmp/cups-$user/classes.conf
 322 touch /tmp/cups-$user/printers.conf
 323
 324 #
 325 # Setup lots of test queues - 500 with PPD files, 500 without...
 326 #
 327
 328 echo "Creating printers.conf for test..."
 329
 330 i=1
 331 while test $i -le $nprinters1; do
 332         cat >>/tmp/cups-$user/printers.conf <<EOF
 333 <Printer test-$i>
 334 Accepting Yes
 335 DeviceURI file:/dev/null
 336 Info Test PS printer $i
 337 JobSheets none none
 338 Location CUPS test suite
 339 State Idle
 340 StateMessage Printer $1 is idle.
 341 </Printer>
 342 EOF
 343
 344         cp testps.ppd /tmp/cups-$user/ppd/test-$i.ppd
 345
 346         i=`expr $i + 1`
 347 done
 348
 349 while test $i -le $nprinters2; do
 350         cat >>/tmp/cups-$user/printers.conf <<EOF
 351 <Printer test-$i>
 352 Accepting Yes
 353 DeviceURI file:/dev/null
 354 Info Test raw printer $i
 355 JobSheets none none
 356 Location CUPS test suite
 357 State Idle
 358 StateMessage Printer $1 is idle.
 359 </Printer>
 360 EOF
 361
 362         i=`expr $i + 1`
 363 done
 364
 365 cp /tmp/cups-$user/printers.conf /tmp/cups-$user/printers.conf.orig
 366
 367 cp $root/conf/mime.types /tmp/cups-$user/mime.types
 368 cp $root/conf/mime.convs /tmp/cups-$user/mime.convs
 369
 370 #
 371 # Setup the paths...
 372 #
 373
 374 echo "Setting up environment variables for test..."
 375
 376 if test "x$LD_LIBRARY_PATH" = x; then
 377         LD_LIBRARY_PATH="$root/cups:$root/filter"
 378 else
 379         LD_LIBRARY_PATH="$root/cups:$root/filter:$LD_LIBRARY_PATH"
 380 fi
 381
 382 export LD_LIBRARY_PATH
 383
 384 LD_PRELOAD="$root/cups/libcups.so.2:$root/filter/libcupsimage.so.2"
 385 export LD_PRELOAD
 386
 387 if test "x$DYLD_LIBRARY_PATH" = x; then
 388         DYLD_LIBRARY_PATH="$root/cups:$root/filter"
 389 else
 390         DYLD_LIBRARY_PATH="$root/cups:$root/filter:$DYLD_LIBRARY_PATH"
 391 fi
 392
 393 export DYLD_LIBRARY_PATH
 394
 395 if test "x$SHLIB_PATH" = x; then
 396         SHLIB_PATH="$root/cups:$root/filter"
 397 else
 398         SHLIB_PATH="$root/cups:$root/filter:$SHLIB_PATH"
 399 fi
 400
 401 export SHLIB_PATH
 402
 403 CUPS_SERVER=localhost; export CUPS_SERVER
 404 CUPS_SERVERROOT=/tmp/cups-$user; export CUPS_SERVERROOT
 405 CUPS_STATEDIR=/tmp/cups-$user; export CUPS_STATEDIR
 406 CUPS_DATADIR=/tmp/cups-$user/share; export CUPS_DATADIR
 407
 408 #
 409 # Set a new home directory to avoid getting user options mixed in...
 410 #
 411
 412 HOME=/tmp/cups-$user
 413 export HOME
 414
 415 #
 416 # Force POSIX locale for tests...
 417 #
 418
 419 LANG=C
 420 export LANG
 421
 422 #
 423 # Start the server; run as foreground daemon in the background...
 424 #
 425
 426 echo "Starting scheduler:"
 427 echo "    $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log 2>&1 &"
 428 echo ""
 429
 430 $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log 2>&1 &
 431 cupsd=$!
 432
 433 #if test -x /usr/bin/strace; then
 434 #       # Trace system calls in cupsd if we have strace...
 435 #       /usr/bin/strace -tt -o /tmp/cups-$user/log/cupsd.trace -p $cupsd &
 436 #fi
 437
 438 if test "x$testtype" = x0; then
 439         echo "Scheduler is PID $cupsd and is listening on port 8631."
 440         echo ""
 441         echo "Set the IPP_PORT environment variable to 8631 to test the software"
 442         echo "interactively from the command-line."
 443         exit 0
 444 fi
 445
 446 if test $argcount -eq 0; then
 447         echo "Scheduler is PID $cupsd; run debugger now if you need to."
 448         echo ""
 449         echo $ac_n "Press ENTER to continue... $ac_c"
 450         read junk
 451 else
 452         echo "Scheduler is PID $cupsd."
 453         sleep 2
 454 fi
 455
 456 IPP_PORT=$port; export IPP_PORT
 457
 458 while true; do
 459         running=`../systemv/lpstat -r 2>/dev/null`
 460         if test "x$running" = "xscheduler is running"; then
 461                 break
 462         fi
 463
 464         echo "Waiting for scheduler to become ready..."
 465         sleep 10
 466 done
 467
 468 #
 469 # Create the test report source file...
 470 #
 471
 472 strfile=cups-str-1.3-`date +%Y-%m-%d`-$user.html
 473
 474 rm -f $strfile
 475 cat str-header.html >$strfile
 476
 477 #
 478 # Run the IPP tests...
 479 #
 480
 481 echo ""
 482 echo "Running IPP compliance tests..."
 483
 484 echo "<H1>1 - IPP Compliance Tests</H1>" >>$strfile
 485 echo "<P>This section provides the results to the IPP compliance tests" >>$strfile
 486 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 487 echo `date "+%Y-%m-%d"` by $user on `hostname`. >>$strfile
 488 echo "<PRE>" >>$strfile
 489
 490 fail=0
 491 for file in 4*.test; do
 492         echo "Performing $file..."
 493         echo "" >>$strfile
 494
 495         ./ipptest ipp://localhost:$port/printers $file >>$strfile
 496         status=$?
 497
 498         if test $status != 0; then
 499                 echo Test failed.
 500                 fail=`expr $fail + 1`
 501         fi
 502 done
 503
 504 echo "</PRE>" >>$strfile
 505
 506 #
 507 # Run the command tests...
 508 #
 509
 510 echo ""
 511 echo "Running command tests..."
 512
 513 echo "<H1>2 - Command Tests</H1>" >>$strfile
 514 echo "<P>This section provides the results to the command tests" >>$strfile
 515 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 516 echo `date "+%Y-%m-%d"` by $user on `hostname`. >>$strfile
 517 echo "<PRE>" >>$strfile
 518
 519 for file in 5*.sh; do
 520         echo "Performing $file..."
 521         echo "" >>$strfile
 522         echo "\"$file\":" >>$strfile
 523
 524         sh $file $pjobs >>$strfile
 525         status=$?
 526
 527         if test $status != 0; then
 528                 echo Test failed.
 529                 fail=`expr $fail + 1`
 530         fi
 531 done
 532
 533 echo "</PRE>" >>$strfile
 534
 535 #
 536 # Wait for jobs to complete...
 537 #
 538
 539 while true; do
 540         jobs=`../systemv/lpstat 2>/dev/null`
 541         if test "x$jobs" = "x"; then
 542                 break
 543         fi
 544
 545         echo "Waiting for jobs to complete..."
 546         sleep 10
 547 done
 548
 549 #
 550 # Stop the server...
 551 #
 552
 553 kill $cupsd
 554
 555 #
 556 # Append the log files for post-mortim...
 557 #
 558
 559 echo "<H1>3 - Log Files</H1>" >>$strfile
 560
 561 echo "<H2>access_log</H2>" >>$strfile
 562 echo "<PRE>" >>$strfile
 563 cat /tmp/cups-$user/log/access_log >>$strfile
 564 echo "</PRE>" >>$strfile
 565
 566 echo "<H2>error_log</H2>" >>$strfile
 567 echo "<PRE>" >>$strfile
 568 cat /tmp/cups-$user/log/error_log >>$strfile
 569 echo "</PRE>" >>$strfile
 570
 571 echo "<H2>page_log</H2>" >>$strfile
 572 echo "<PRE>" >>$strfile
 573 cat /tmp/cups-$user/log/page_log >>$strfile
 574 echo "</PRE>" >>$strfile
 575
 576 if test -f /tmp/cups-$user/log/cupsd.trace; then
 577         echo "<H2>cupsd.trace</H2>" >>$strfile
 578         echo "<PRE>" >>$strfile
 579         cat /tmp/cups-$user/log/cupsd.trace >>$strfile
 580         echo "</PRE>" >>$strfile
 581 fi
 582
 583 #
 584 # Format the reports and tell the user where to find them...
 585 #
 586
 587 cat str-trailer.html >>$strfile
 588
 589 echo ""
 590
 591 if test $fail != 0; then
 592         echo "$fail tests failed."
 593 else
 594         echo "All tests were successful."
 595 fi
 596
 597 echo "Log files can be found in /tmp/cups-$user/log."
 598 echo "A HTML report was created in test/$strfile."
 599 echo ""
 600
 601 #
 602 # End of "$Id: run-stp-tests.sh 6649 2007-07-11 21:46:42Z mike $"
 603 #