test/run-stp-tests.sh

   1 #!/bin/sh
   2 #
   3 # "$Id: run-stp-tests.sh 6297 2007-02-21 02:24:16Z mike $"
   4 #
   5 #   Perform the complete set of IPP compliance tests specified in the
   6 #   CUPS Software Test Plan.
   7 #
   8 #   Copyright 1997-2006 by Easy Software Products, all rights reserved.
   9 #
  10 #   These coded instructions, statements, and computer programs are the
  11 #   property of Easy Software Products and are protected by Federal
  12 #   copyright law.  Distribution and use rights are outlined in the file
  13 #   "LICENSE.txt" which should have been included with this file.  If this
  14 #   file is missing or damaged please contact Easy Software Products
  15 #   at:
  16 #
  17 #       Attn: CUPS Licensing Information
  18 #       Easy Software Products
  19 #       44141 Airport View Drive, Suite 204
  20 #       Hollywood, Maryland 20636 USA
  21 #
  22 #       Voice: (301) 373-9600
  23 #       EMail: cups-info@cups.org
  24 #         WWW: http://www.cups.org
  25 #
  26
  27 argcount=$#
  28
  29 #
  30 # Make the IPP test program...
  31 #
  32
  33 make
  34
  35 #
  36 # Figure out the proper echo options...
  37 #
  38
  39 if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null; then
  40         ac_n=-n
  41         ac_c=
  42 else
  43         ac_n=
  44         ac_c='\c'
  45 fi
  46
  47 #
  48 # Greet the tester...
  49 #
  50
  51 echo "Welcome to the CUPS Automated Test Script."
  52 echo ""
  53 echo "Before we begin, it is important that you understand that the larger"
  54 echo "tests require significant amounts of RAM and disk space.  If you"
  55 echo "attempt to run one of the big tests on a system that lacks sufficient"
  56 echo "disk and virtual memory, the UNIX kernel might decide to kill one or"
  57 echo "more system processes that you've grown attached to, like the X"
  58 echo "server.  The question you may want to ask yourself before running a"
  59 echo "large test is: Do you feel lucky?"
  60 echo ""
  61 echo "OK, now that we have the Dirty Harry quote out of the way, please"
  62 echo "choose the type of test you wish to perform:"
  63 echo ""
  64 echo "0 - No testing, keep the scheduler running for me (all systems)"
  65 echo "1 - Basic conformance test, no load testing (all systems)"
  66 echo "2 - Basic conformance test, some load testing (minimum 256MB VM, 50MB disk)"
  67 echo "3 - Basic conformance test, extreme load testing (minimum 1GB VM, 500MB disk)"
  68 echo "4 - Basic conformance test, torture load testing (minimum 2GB VM, 1GB disk)"
  69 echo ""
  70 echo $ac_n "Enter the number of the test you wish to perform: [1] $ac_c"
  71
  72 if test $# -gt 0; then
  73         testtype=$1
  74         shift
  75 else
  76         read testtype
  77 fi
  78 echo ""
  79
  80 case "$testtype" in
  81         0)
  82                 echo "Running in test mode (0)"
  83                 nprinters1=0
  84                 nprinters2=0
  85                 pjobs=0
  86                 ;;
  87         2)
  88                 echo "Running the medium tests (2)"
  89                 nprinters1=10
  90                 nprinters2=20
  91                 pjobs=20
  92                 ;;
  93         3)
  94                 echo "Running the extreme tests (3)"
  95                 nprinters1=500
  96                 nprinters2=1000
  97                 pjobs=100
  98                 ;;
  99         4)
 100                 echo "Running the torture tests (4)"
 101                 nprinters1=10000
 102                 nprinters2=20000
 103                 pjobs=200
 104                 ;;
 105         *)
 106                 echo "Running the timid tests (1)"
 107                 nprinters1=0
 108                 nprinters2=0
 109                 pjobs=0
 110                 ;;
 111 esac
 112
 113 #
 114 # See if we want to do SSL testing...
 115 #
 116
 117 echo ""
 118 echo "Now you can choose whether to create a SSL/TLS encryption key and"
 119 echo "certificate for testing; these tests currently require the OpenSSL"
 120 echo "tools:"
 121 echo ""
 122 echo "0 - Do not do SSL/TLS encryption tests"
 123 echo "1 - Test but do not require encryption"
 124 echo "2 - Test and require encryption"
 125 echo ""
 126 echo $ac_n "Enter the number of the SSL/TLS tests to perform: [0] $ac_c"
 127
 128 if test $# -gt 0; then
 129         ssltype=$1
 130         shift
 131 else
 132         read ssltype
 133 fi
 134 echo ""
 135
 136 case "$ssltype" in
 137         1)
 138                 echo "Will test but not require encryption (1)"
 139                 ;;
 140         2)
 141                 echo "Will test and require encryption (2)"
 142                 ;;
 143         *)
 144                 echo "Not using SSL/TLS (0)"
 145                 ssltype=0
 146                 ;;
 147 esac
 148
 149 #
 150 # Information for the server/tests...
 151 #
 152
 153 user="$USER"
 154 if test -z "$user"; then
 155         if test -x /usr/ucb/whoami; then
 156                 user=`/usr/ucb/whoami`
 157         else
 158                 user=`whoami`
 159         fi
 160
 161         if test -z "$user"; then
 162                 user="unknown"
 163         fi
 164 fi
 165
 166 port=8631
 167 cwd=`pwd`
 168 root=`dirname $cwd`
 169
 170 #
 171 # Make sure that the LPDEST and PRINTER environment variables are
 172 # not included in the environment that is passed to the tests.  These
 173 # will usually cause tests to fail erroneously...
 174 #
 175
 176 typeset +x LPDEST
 177 typeset +x PRINTER
 178
 179 #
 180 # See if we want to use valgrind...
 181 #
 182
 183 echo ""
 184 echo "This test script can use the Valgrind software from:"
 185 echo ""
 186 echo "    http://developer.kde.org/~sewardj/"
 187 echo ""
 188 echo $ac_n "Enter Y to use Valgrind or N to not use Valgrind: [N] $ac_c"
 189
 190 if test $# -gt 0; then
 191         usevalgrind=$1
 192         shift
 193 else
 194         read usevalgrind
 195 fi
 196 echo ""
 197
 198 case "$usevalgrind" in
 199         Y* | y*)
 200                 valgrind="valgrind --tool=memcheck --log-file=/tmp/cups-$user/log/valgrind --error-limit=no --leak-check=yes --trace-children=yes"
 201                 echo "Using Valgrind; log files can be found in /tmp/cups-$user/log..."
 202                 ;;
 203
 204         *)
 205                 valgrind=""
 206                 ;;
 207 esac
 208
 209 #
 210 # Start by creating temporary directories for the tests...
 211 #
 212
 213 echo "Creating directories for test..."
 214
 215 rm -rf /tmp/cups-$user
 216 mkdir /tmp/cups-$user
 217 mkdir /tmp/cups-$user/bin
 218 mkdir /tmp/cups-$user/bin/backend
 219 mkdir /tmp/cups-$user/bin/filter
 220 mkdir /tmp/cups-$user/certs
 221 mkdir /tmp/cups-$user/share
 222 mkdir /tmp/cups-$user/share/banners
 223 mkdir /tmp/cups-$user/share/model
 224 mkdir /tmp/cups-$user/interfaces
 225 mkdir /tmp/cups-$user/log
 226 mkdir /tmp/cups-$user/ppd
 227 mkdir /tmp/cups-$user/spool
 228 mkdir /tmp/cups-$user/spool/temp
 229 mkdir /tmp/cups-$user/ssl
 230
 231 ln -s $root/backend/http /tmp/cups-$user/bin/backend
 232 ln -s $root/backend/ipp /tmp/cups-$user/bin/backend
 233 ln -s $root/backend/lpd /tmp/cups-$user/bin/backend
 234 ln -s $root/backend/parallel /tmp/cups-$user/bin/backend
 235 ln -s $root/backend/serial /tmp/cups-$user/bin/backend
 236 ln -s $root/backend/snmp /tmp/cups-$user/bin/backend
 237 ln -s $root/backend/socket /tmp/cups-$user/bin/backend
 238 ln -s $root/backend/usb /tmp/cups-$user/bin/backend
 239 ln -s $root/cgi-bin /tmp/cups-$user/bin
 240 ln -s $root/monitor /tmp/cups-$user/bin
 241 ln -s $root/notifier /tmp/cups-$user/bin
 242 ln -s $root/scheduler /tmp/cups-$user/bin/daemon
 243 ln -s $root/filter/hpgltops /tmp/cups-$user/bin/filter
 244 ln -s $root/filter/imagetops /tmp/cups-$user/bin/filter
 245 ln -s $root/filter/imagetoraster /tmp/cups-$user/bin/filter
 246 ln -s $root/filter/pstops /tmp/cups-$user/bin/filter
 247 ln -s $root/filter/rastertoepson /tmp/cups-$user/bin/filter
 248 ln -s $root/filter/rastertohp /tmp/cups-$user/bin/filter
 249 ln -s $root/filter/texttops /tmp/cups-$user/bin/filter
 250 ln -s $root/pdftops/pdftops /tmp/cups-$user/bin/filter
 251
 252 ln -s $root/data/classified /tmp/cups-$user/share/banners
 253 ln -s $root/data/confidential /tmp/cups-$user/share/banners
 254 ln -s $root/data/secret /tmp/cups-$user/share/banners
 255 ln -s $root/data/standard /tmp/cups-$user/share/banners
 256 ln -s $root/data/topsecret /tmp/cups-$user/share/banners
 257 ln -s $root/data/unclassified /tmp/cups-$user/share/banners
 258 ln -s $root/data /tmp/cups-$user/share/charmaps
 259 ln -s $root/data /tmp/cups-$user/share/charsets
 260 ln -s $root/data /tmp/cups-$user/share
 261 ln -s $root/fonts /tmp/cups-$user/share
 262 ln -s $root/ppd/*.ppd /tmp/cups-$user/share/model
 263 ln -s $root/templates /tmp/cups-$user/share
 264
 265 if test $ssltype != 0; then
 266         mkdir $root/ssl
 267         cp server.* $root/ssl
 268 fi
 269
 270 #
 271 # Then create the necessary config files...
 272 #
 273
 274 echo "Creating cupsd.conf for test..."
 275
 276 if test $ssltype = 2; then
 277         encryption="Encryption Required"
 278 else
 279         encryption=""
 280 fi
 281
 282 cat >/tmp/cups-$user/cupsd.conf <<EOF
 283 Browsing Off
 284 FileDevice yes
 285 Printcap
 286 Listen 127.0.0.1:$port
 287 User $user
 288 ServerRoot /tmp/cups-$user
 289 StateDir /tmp/cups-$user
 290 ServerBin /tmp/cups-$user/bin
 291 CacheDir /tmp/cups-$user/share
 292 DataDir /tmp/cups-$user/share
 293 FontPath /tmp/cups-$user/share/fonts
 294 DocumentRoot $root/doc
 295 RequestRoot /tmp/cups-$user/spool
 296 TempDir /tmp/cups-$user/spool/temp
 297 MaxLogSize 0
 298 AccessLog /tmp/cups-$user/log/access_log
 299 ErrorLog /tmp/cups-$user/log/error_log
 300 PageLog /tmp/cups-$user/log/page_log
 301 LogLevel debug2
 302 PreserveJobHistory Yes
 303 <Policy default>
 304 <Limit All>
 305 Order Deny,Allow
 306 Deny from all
 307 Allow from 127.0.0.1
 308 $encryption
 309 </Limit>
 310 </Policy>
 311 EOF
 312
 313 touch /tmp/cups-$user/classes.conf
 314 touch /tmp/cups-$user/printers.conf
 315
 316 #
 317 # Setup lots of test queues - 500 with PPD files, 500 without...
 318 #
 319
 320 echo "Creating printers.conf for test..."
 321
 322 i=1
 323 while test $i -le $nprinters1; do
 324         cat >>/tmp/cups-$user/printers.conf <<EOF
 325 <Printer test-$i>
 326 Accepting Yes
 327 DeviceURI file:/dev/null
 328 Info Test PS printer $i
 329 JobSheets none none
 330 Location CUPS test suite
 331 State Idle
 332 StateMessage Printer $1 is idle.
 333 </Printer>
 334 EOF
 335
 336         cp testps.ppd /tmp/cups-$user/ppd/test-$i.ppd
 337
 338         i=`expr $i + 1`
 339 done
 340
 341 while test $i -le $nprinters2; do
 342         cat >>/tmp/cups-$user/printers.conf <<EOF
 343 <Printer test-$i>
 344 Accepting Yes
 345 DeviceURI file:/dev/null
 346 Info Test raw printer $i
 347 JobSheets none none
 348 Location CUPS test suite
 349 State Idle
 350 StateMessage Printer $1 is idle.
 351 </Printer>
 352 EOF
 353
 354         i=`expr $i + 1`
 355 done
 356
 357 cp /tmp/cups-$user/printers.conf /tmp/cups-$user/printers.conf.orig
 358
 359 cp $root/conf/mime.types /tmp/cups-$user/mime.types
 360 cp $root/conf/mime.convs /tmp/cups-$user/mime.convs
 361
 362 #
 363 # Setup the paths...
 364 #
 365
 366 echo "Setting up environment variables for test..."
 367
 368 if test "x$LD_LIBRARY_PATH" = x; then
 369         LD_LIBRARY_PATH="$root/cups:$root/filter"
 370 else
 371         LD_LIBRARY_PATH="$root/cups:$root/filter:$LD_LIBRARY_PATH"
 372 fi
 373
 374 export LD_LIBRARY_PATH
 375
 376 LD_PRELOAD="$root/cups/libcups.so.2:$root/filter/libcupsimage.so.2"
 377 export LD_PRELOAD
 378
 379 if test "x$DYLD_LIBRARY_PATH" = x; then
 380         DYLD_LIBRARY_PATH="$root/cups:$root/filter"
 381 else
 382         DYLD_LIBRARY_PATH="$root/cups:$root/filter:$DYLD_LIBRARY_PATH"
 383 fi
 384
 385 export DYLD_LIBRARY_PATH
 386
 387 if test "x$SHLIB_PATH" = x; then
 388         SHLIB_PATH="$root/cups:$root/filter"
 389 else
 390         SHLIB_PATH="$root/cups:$root/filter:$SHLIB_PATH"
 391 fi
 392
 393 export SHLIB_PATH
 394
 395 CUPS_SERVER=localhost; export CUPS_SERVER
 396 CUPS_SERVERROOT=/tmp/cups-$user; export CUPS_SERVERROOT
 397 CUPS_STATEDIR=/tmp/cups-$user; export CUPS_STATEDIR
 398 CUPS_DATADIR=/tmp/cups-$user/share; export CUPS_DATADIR
 399
 400 #
 401 # Set a new home directory to avoid getting user options mixed in...
 402 #
 403
 404 HOME=/tmp/cups-$user
 405 export HOME
 406
 407 #
 408 # Start the server; run as foreground daemon in the background...
 409 #
 410
 411 echo "Starting scheduler:"
 412 echo "    $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log 2>&1 &"
 413 echo ""
 414
 415 $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log 2>&1 &
 416 cupsd=$!
 417
 418 #if test -x /usr/bin/strace; then
 419 #       # Trace system calls in cupsd if we have strace...
 420 #       /usr/bin/strace -tt -o /tmp/cups-$user/log/cupsd.trace -p $cupsd &
 421 #fi
 422
 423 if test "x$testtype" = x0; then
 424         echo "Scheduler is PID $cupsd and is listening on port 8631."
 425         echo ""
 426         echo "Set the IPP_PORT environment variable to 8631 to test the software"
 427         echo "interactively from the command-line."
 428         exit 0
 429 fi
 430
 431 if test $argcount -eq 0; then
 432         echo "Scheduler is PID $cupsd; run debugger now if you need to."
 433         echo ""
 434         echo $ac_n "Press ENTER to continue... $ac_c"
 435         read junk
 436 else
 437         echo "Scheduler is PID $cupsd."
 438         sleep 2
 439 fi
 440
 441 IPP_PORT=$port; export IPP_PORT
 442
 443 while true; do
 444         running=`../systemv/lpstat -r 2>/dev/null`
 445         if test "x$running" = "xscheduler is running"; then
 446                 break
 447         fi
 448
 449         echo "Waiting for scheduler to become ready..."
 450         sleep 10
 451 done
 452
 453 #
 454 # Create the test report source file...
 455 #
 456
 457 strfile=cups-str-1.3-`date +%Y-%m-%d`-$user.html
 458
 459 rm -f $strfile
 460 cat str-header.html >$strfile
 461
 462 #
 463 # Run the IPP tests...
 464 #
 465
 466 echo ""
 467 echo "Running IPP compliance tests..."
 468
 469 echo "<H1>1 - IPP Compliance Tests</H1>" >>$strfile
 470 echo "<P>This section provides the results to the IPP compliance tests" >>$strfile
 471 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 472 echo `date "+%Y-%m-%d"` by $user on `hostname`. >>$strfile
 473 echo "<PRE>" >>$strfile
 474
 475 fail=0
 476 for file in 4*.test; do
 477         echo "Performing $file..."
 478         echo "" >>$strfile
 479
 480         ./ipptest ipp://localhost:$port/printers $file >>$strfile
 481         status=$?
 482
 483         if test $status != 0; then
 484                 echo Test failed.
 485                 fail=`expr $fail + 1`
 486         fi
 487 done
 488
 489 echo "</PRE>" >>$strfile
 490
 491 #
 492 # Run the command tests...
 493 #
 494
 495 echo ""
 496 echo "Running command tests..."
 497
 498 echo "<H1>2 - Command Tests</H1>" >>$strfile
 499 echo "<P>This section provides the results to the command tests" >>$strfile
 500 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 501 echo `date "+%Y-%m-%d"` by $user on `hostname`. >>$strfile
 502 echo "<PRE>" >>$strfile
 503
 504 for file in 5*.sh; do
 505         echo "Performing $file..."
 506         echo "" >>$strfile
 507         echo "\"$file\":" >>$strfile
 508
 509         sh $file $pjobs >>$strfile
 510         status=$?
 511
 512         if test $status != 0; then
 513                 echo Test failed.
 514                 fail=`expr $fail + 1`
 515         fi
 516 done
 517
 518 echo "</PRE>" >>$strfile
 519
 520 #
 521 # Wait for jobs to complete...
 522 #
 523
 524 while true; do
 525         jobs=`../systemv/lpstat 2>/dev/null`
 526         if test "x$jobs" = "x"; then
 527                 break
 528         fi
 529
 530         echo "Waiting for jobs to complete..."
 531         sleep 10
 532 done
 533
 534 #
 535 # Stop the server...
 536 #
 537
 538 kill $cupsd
 539
 540 #
 541 # Append the log files for post-mortim...
 542 #
 543
 544 echo "<H1>3 - Log Files</H1>" >>$strfile
 545
 546 echo "<H2>access_log</H2>" >>$strfile
 547 echo "<PRE>" >>$strfile
 548 cat /tmp/cups-$user/log/access_log >>$strfile
 549 echo "</PRE>" >>$strfile
 550
 551 echo "<H2>error_log</H2>" >>$strfile
 552 echo "<PRE>" >>$strfile
 553 cat /tmp/cups-$user/log/error_log >>$strfile
 554 echo "</PRE>" >>$strfile
 555
 556 echo "<H2>page_log</H2>" >>$strfile
 557 echo "<PRE>" >>$strfile
 558 cat /tmp/cups-$user/log/page_log >>$strfile
 559 echo "</PRE>" >>$strfile
 560
 561 if test -f /tmp/cups-$user/log/cupsd.trace; then
 562         echo "<H2>cupsd.trace</H2>" >>$strfile
 563         echo "<PRE>" >>$strfile
 564         cat /tmp/cups-$user/log/cupsd.trace >>$strfile
 565         echo "</PRE>" >>$strfile
 566 fi
 567
 568 #
 569 # Format the reports and tell the user where to find them...
 570 #
 571
 572 cat str-trailer.html >>$strfile
 573
 574 echo ""
 575
 576 if test $fail != 0; then
 577         echo "$fail tests failed."
 578 else
 579         echo "All tests were successful."
 580 fi
 581
 582 echo "Log files can be found in /tmp/cups-$user/log."
 583 echo "A HTML report was created in test/$strfile."
 584 echo ""
 585
 586 #
 587 # End of "$Id: run-stp-tests.sh 6297 2007-02-21 02:24:16Z mike $"
 588 #