test/run-stp-tests.sh

   1 #!/bin/sh
   2 #
   3 # "$Id$"
   4 #
   5 #   Perform the complete set of IPP compliance tests specified in the
   6 #   CUPS Software Test Plan.
   7 #
   8 #   Copyright 1997-2005 by Easy Software Products, all rights reserved.
   9 #
  10 #   These coded instructions, statements, and computer programs are the
  11 #   property of Easy Software Products and are protected by Federal
  12 #   copyright law.  Distribution and use rights are outlined in the file
  13 #   "LICENSE.txt" which should have been included with this file.  If this
  14 #   file is missing or damaged please contact Easy Software Products
  15 #   at:
  16 #
  17 #       Attn: CUPS Licensing Information
  18 #       Easy Software Products
  19 #       44141 Airport View Drive, Suite 204
  20 #       Hollywood, Maryland 20636 USA
  21 #
  22 #       Voice: (301) 373-9600
  23 #       EMail: cups-info@cups.org
  24 #         WWW: http://www.cups.org
  25 #
  26
  27 #
  28 # Make the IPP test program...
  29 #
  30
  31 make
  32
  33 #
  34 # Greet the tester...
  35 #
  36
  37 echo "Welcome to the CUPS Automated Test Script."
  38 echo ""
  39 echo "Before we begin, it is important that you understand that the larger"
  40 echo "tests require significant amounts of RAM and disk space.  If you"
  41 echo "attempt to run one of the big tests on a system that lacks sufficient"
  42 echo "disk and virtual memory, the UNIX kernel might decide to kill one or"
  43 echo "more system processes that you've grown attached to, like the X"
  44 echo "server.  The question you may want to ask yourself before running a"
  45 echo "large test is: Do you feel lucky?"
  46 echo ""
  47 echo "OK, now that we have the Dirty Harry quote out of the way, please"
  48 echo "choose the type of test you wish to perform:"
  49 echo ""
  50 echo "0 - No testing, keep the scheduler running for me (all systems)"
  51 echo "1 - Basic conformance test, no load testing (all systems)"
  52 echo "2 - Basic conformance test, some load testing (minimum 256MB VM, 50MB disk)"
  53 echo "3 - Basic conformance test, extreme load testing (minimum 1GB VM, 500MB disk)"
  54 echo "4 - Basic conformance test, torture load testing (minimum 2GB VM, 1GB disk)"
  55 echo ""
  56 echo "Please enter the number of the test you wish to perform:"
  57
  58 read testtype
  59
  60 case "$testtype" in
  61         0)
  62                 echo "Running in test mode (0)"
  63                 nprinters1=0
  64                 nprinters2=0
  65                 pjobs=0
  66                 ;;
  67         2)
  68                 echo "Running the medium tests (2)"
  69                 nprinters1=10
  70                 nprinters2=20
  71                 pjobs=20
  72                 ;;
  73         3)
  74                 echo "Running the extreme tests (3)"
  75                 nprinters1=500
  76                 nprinters2=1000
  77                 pjobs=100
  78                 ;;
  79         4)
  80                 echo "Running the torture tests (4)"
  81                 nprinters1=10000
  82                 nprinters2=20000
  83                 pjobs=200
  84                 ;;
  85         *)
  86                 echo "Running the timid tests (1)"
  87                 nprinters1=0
  88                 nprinters2=0
  89                 pjobs=0
  90                 ;;
  91 esac
  92
  93 #
  94 # See if we want to do SSL testing...
  95 #
  96
  97 echo ""
  98 echo "Now you can choose whether to create a SSL/TLS encryption key and"
  99 echo "certificate for testing; these tests currently require the OpenSSL"
 100 echo "tools:"
 101 echo ""
 102 echo "0 - Do not do SSL/TLS encryption tests"
 103 echo "1 - Create a SSL/TLS certificate and key, but do not require encryption"
 104 echo "2 - Create a SSL/TLS certificate and key and require encryption"
 105 echo ""
 106
 107 read ssltype
 108
 109 case "$ssltype" in
 110         1 | 2)
 111                 if test -f server.key; then
 112                         echo "Using existing SSL/TLS certificate and key..."
 113                 else
 114                         echo "Generating SSL/TLS certificate and key..."
 115                         openssl req -new -x509 -keyout server.key -out server.crt -days 3650 -nodes >/dev/null <<EOF
 116 ZZ
 117 Testland
 118 Testville
 119 Test Company
 120
 121 `hostname`
 122 test@domain.com
 123 EOF
 124                 fi
 125                 ;;
 126
 127         *)
 128                 echo "Not using SSL/TLS ($ssltype)..."
 129                 ssltype=0
 130                 ;;
 131 esac
 132
 133 #
 134 # Information for the server/tests...
 135 #
 136
 137 user=`whoami`
 138 port=8631
 139 cwd=`pwd`
 140 root=`dirname $cwd`
 141
 142 #
 143 # Make sure that the LPDEST and PRINTER environment variables are
 144 # not included in the environment that is passed to the tests.  These
 145 # will usually cause tests to fail erroneously...
 146 #
 147
 148 typeset +x LPDEST
 149 typeset +x PRINTER
 150
 151 #
 152 # See if we want to use valgrind...
 153 #
 154
 155 echo ""
 156 echo "This test script can use the Valgrind software from:"
 157 echo ""
 158 echo "    http://developer.kde.org/~sewardj/"
 159 echo ""
 160 echo "Please enter Y to use Valgrind or N to not use Valgrind:"
 161
 162 read usevalgrind
 163
 164 case "$usevalgrind" in
 165         Y* | y*)
 166                 valgrind="valgrind --tool=memcheck --logfile=/tmp/$user/log/valgrind --error-limit=no --leak-check=yes --trace-children=yes"
 167                 echo "Using Valgrind; log files can be found in /tmp/$user/log..."
 168                 ;;
 169
 170         *)
 171                 valgrind=""
 172                 ;;
 173 esac
 174
 175 #
 176 # Start by creating temporary directories for the tests...
 177 #
 178
 179 rm -rf /tmp/$user
 180 mkdir /tmp/$user
 181 mkdir /tmp/$user/bin
 182 mkdir /tmp/$user/bin/backend
 183 mkdir /tmp/$user/bin/filter
 184 mkdir /tmp/$user/certs
 185 mkdir /tmp/$user/share
 186 mkdir /tmp/$user/share/banners
 187 mkdir /tmp/$user/share/model
 188 mkdir /tmp/$user/interfaces
 189 mkdir /tmp/$user/log
 190 mkdir /tmp/$user/ppd
 191 mkdir /tmp/$user/spool
 192 mkdir /tmp/$user/spool/temp
 193 mkdir /tmp/$user/ssl
 194
 195 ln -s $root/backend/http /tmp/$user/bin/backend
 196 ln -s $root/backend/ipp /tmp/$user/bin/backend
 197 ln -s $root/backend/lpd /tmp/$user/bin/backend
 198 ln -s $root/backend/parallel /tmp/$user/bin/backend
 199 ln -s $root/backend/serial /tmp/$user/bin/backend
 200 ln -s $root/backend/socket /tmp/$user/bin/backend
 201 ln -s $root/backend/usb /tmp/$user/bin/backend
 202 ln -s $root/cgi-bin /tmp/$user/bin
 203 ln -s $root/scheduler /tmp/$user/bin/daemon
 204 ln -s $root/filter/hpgltops /tmp/$user/bin/filter
 205 ln -s $root/filter/imagetops /tmp/$user/bin/filter
 206 ln -s $root/filter/imagetoraster /tmp/$user/bin/filter
 207 ln -s $root/filter/pstops /tmp/$user/bin/filter
 208 ln -s $root/filter/rastertoepson /tmp/$user/bin/filter
 209 ln -s $root/filter/rastertohp /tmp/$user/bin/filter
 210 ln -s $root/filter/texttops /tmp/$user/bin/filter
 211 ln -s $root/pdftops/pdftops /tmp/$user/bin/filter
 212
 213 ln -s $root/data/classified /tmp/$user/share/banners
 214 ln -s $root/data/confidential /tmp/$user/share/banners
 215 ln -s $root/data/secret /tmp/$user/share/banners
 216 ln -s $root/data/standard /tmp/$user/share/banners
 217 ln -s $root/data/topsecret /tmp/$user/share/banners
 218 ln -s $root/data/unclassified /tmp/$user/share/banners
 219 ln -s $root/data /tmp/$user/share/charsets
 220 ln -s $root/data /tmp/$user/share
 221 ln -s $root/fonts /tmp/$user/share
 222 ln -s $root/ppd/*.ppd /tmp/$user/share/model
 223 ln -s $root/templates /tmp/$user/share
 224
 225 if test $ssltype != 0; then
 226         mkdir $root/ssl
 227         cp server.* $root/ssl
 228 fi
 229
 230 #
 231 # Then create the necessary config files...
 232 #
 233
 234 if test $ssltype = 2; then
 235         encryption="Encryption Required"
 236 else
 237         encryption=""
 238 fi
 239
 240 cat >/tmp/$user/cupsd.conf <<EOF
 241 Browsing Off
 242 FileDevice yes
 243 Printcap
 244 Listen 127.0.0.1:$port
 245 User $user
 246 ServerRoot /tmp/$user
 247 StateDir /tmp/$user
 248 ServerBin /tmp/$user/bin
 249 CacheDir /tmp/$user/share
 250 DataDir /tmp/$user/share
 251 FontPath /tmp/$user/share/fonts
 252 DocumentRoot $root/doc
 253 RequestRoot /tmp/$user/spool
 254 TempDir /tmp/$user/spool/temp
 255 MaxLogSize 0
 256 AccessLog /tmp/$user/log/access_log
 257 ErrorLog /tmp/$user/log/error_log
 258 PageLog /tmp/$user/log/page_log
 259 LogLevel debug
 260 PreserveJobHistory Yes
 261 <Location />
 262 Order deny,allow
 263 Deny from all
 264 Allow from 127.0.0.1
 265 $encryption
 266 </Location>
 267 <Location /admin>
 268 Order deny,allow
 269 Deny from all
 270 Allow from 127.0.0.1
 271 Require valid-user
 272 $encryption
 273 </Location>
 274 <Policy default>
 275 <Limit All>
 276 Order Deny,Allow
 277 </Limit>
 278 </Policy>
 279 EOF
 280
 281 touch /tmp/$user/classes.conf
 282 touch /tmp/$user/printers.conf
 283
 284 #
 285 # Setup lots of test queues - 500 with PPD files, 500 without...
 286 #
 287
 288 i=1
 289 while test $i -le $nprinters1; do
 290         cat >>/tmp/$user/printers.conf <<EOF
 291 <Printer test-$i>
 292 Accepting Yes
 293 DeviceURI file:/dev/null
 294 Info Test PS printer $i
 295 JobSheets none none
 296 Location CUPS test suite
 297 State Idle
 298 StateMessage Printer $1 is idle.
 299 </Printer>
 300 EOF
 301
 302         cp testps.ppd /tmp/$user/ppd/test-$i.ppd
 303
 304         i=`expr $i + 1`
 305 done
 306
 307 while test $i -le $nprinters2; do
 308         cat >>/tmp/$user/printers.conf <<EOF
 309 <Printer test-$i>
 310 Accepting Yes
 311 DeviceURI file:/dev/null
 312 Info Test raw printer $i
 313 JobSheets none none
 314 Location CUPS test suite
 315 State Idle
 316 StateMessage Printer $1 is idle.
 317 </Printer>
 318 EOF
 319
 320         i=`expr $i + 1`
 321 done
 322
 323 cp /tmp/$user/printers.conf /tmp/$user/printers.conf.orig
 324
 325 cp $root/conf/mime.types /tmp/$user/mime.types
 326 cp $root/conf/mime.convs /tmp/$user/mime.convs
 327
 328 #
 329 # Setup the paths...
 330 #
 331
 332 if test "x$LD_LIBRARY_PATH" = x; then
 333         LD_LIBRARY_PATH="$root/cups:$root/filter"
 334 else
 335         LD_LIBRARY_PATH="$root/cups:$root/filter:$LD_LIBRARY_PATH"
 336 fi
 337
 338 export LD_LIBRARY_PATH
 339
 340 LD_PRELOAD="$root/cups/libcups.so.2:$root/filter/libcupsimage.so.2"
 341 export LD_PRELOAD
 342
 343 if test "x$DYLD_LIBRARY_PATH" = x; then
 344         DYLD_LIBRARY_PATH="$root/cups:$root/filter"
 345 else
 346         DYLD_LIBRARY_PATH="$root/cups:$root/filter:$DYLD_LIBRARY_PATH"
 347 fi
 348
 349 export DYLD_LIBRARY_PATH
 350
 351 if test "x$SHLIB_PATH" = x; then
 352         SHLIB_PATH="$root/cups:$root/filter"
 353 else
 354         SHLIB_PATH="$root/cups:$root/filter:$SHLIB_PATH"
 355 fi
 356
 357 export SHLIB_PATH
 358
 359 CUPS_SERVERROOT=/tmp/$user; export CUPS_SERVERROOT
 360 CUPS_STATEDIR=/tmp/$user; export CUPS_STATEDIR
 361 CUPS_DATADIR=/tmp/$user/share; export CUPS_DATADIR
 362
 363 #
 364 # Set a new home directory to avoid getting user options mixed in...
 365 #
 366
 367 HOME=/tmp/$user
 368 export HOME
 369
 370 #
 371 # Start the server; run as foreground daemon in the background...
 372 #
 373
 374 echo "Starting scheduler:"
 375 echo "    $valgrind ../scheduler/cupsd -c /tmp/$user/cupsd.conf -f >/tmp/$user/log/debug_log &"
 376 echo ""
 377
 378 $valgrind ../scheduler/cupsd -c /tmp/$user/cupsd.conf -f >/tmp/$user/log/debug_log &
 379 cupsd=$!
 380
 381 #if test -x /usr/bin/strace; then
 382 #       # Trace system calls in cupsd if we have strace...
 383 #       /usr/bin/strace -tt -o /tmp/$user/log/cupsd.trace -p $cupsd &
 384 #fi
 385
 386 if test "x$testtype" = x0; then
 387         echo "Scheduler is PID $cupsd and is listening on port 8631."
 388         echo ""
 389         echo "Set the IPP_PORT environment variable to 8631 to test the software"
 390         echo "interactively from the command-line."
 391         exit 0
 392 fi
 393
 394 echo "Scheduler is PID $cupsd; run debugger now if you need to."
 395 echo ""
 396 echo "Press ENTER to continue..."
 397 read junk
 398
 399 IPP_PORT=$port; export IPP_PORT
 400
 401 while true; do
 402         running=`../systemv/lpstat -r 2>/dev/null`
 403         if test "x$running" = "xscheduler is running"; then
 404                 break
 405         fi
 406
 407         echo "Waiting for scheduler to become ready..."
 408         sleep 10
 409 done
 410
 411 #
 412 # Create the test report source file...
 413 #
 414
 415 strfile=cups-str-1.2-`date +%Y-%m-%d`-`whoami`.shtml
 416
 417 rm -f $strfile
 418 cat str-header.html >$strfile
 419
 420 #
 421 # Run the IPP tests...
 422 #
 423
 424 echo "Running IPP compliance tests..."
 425
 426 echo "<H1>IPP Compliance Tests</H1>" >>$strfile
 427 echo "<P>This section provides the results to the IPP compliance tests" >>$strfile
 428 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 429 echo `date "+%Y-%m-%d"` by `whoami` on `hostname`. >>$strfile
 430 echo "<PRE>" >>$strfile
 431
 432 fail=0
 433 for file in 4*.test; do
 434         echo "Performing $file..."
 435         echo "" >>$strfile
 436
 437         ./ipptest ipp://localhost:$port/printers $file >>$strfile
 438         status=$?
 439
 440         if test $status != 0; then
 441                 echo Test failed.
 442                 fail=`expr $fail + 1`
 443         fi
 444 done
 445
 446 echo "</PRE>" >>$strfile
 447
 448 #
 449 # Run the command tests...
 450 #
 451
 452 echo "Running command tests..."
 453
 454 echo "<H1>Command Tests</H1>" >>$strfile
 455 echo "<P>This section provides the results to the command tests" >>$strfile
 456 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 457 echo `date "+%Y-%m-%d"` by `whoami` on `hostname`. >>$strfile
 458 echo "<PRE>" >>$strfile
 459
 460 for file in 5*.sh; do
 461         echo "Performing $file..."
 462         echo "" >>$strfile
 463         echo "\"$file\":" >>$strfile
 464
 465         sh $file $pjobs >>$strfile
 466         status=$?
 467
 468         if test $status != 0; then
 469                 echo Test failed.
 470                 fail=`expr $fail + 1`
 471         fi
 472 done
 473
 474 echo "</PRE>" >>$strfile
 475
 476 #
 477 # Wait for jobs to complete...
 478 #
 479
 480 while true; do
 481         jobs=`../systemv/lpstat 2>/dev/null`
 482         if test "x$jobs" = "x"; then
 483                 break
 484         fi
 485
 486         echo "Waiting for jobs to complete..."
 487         sleep 10
 488 done
 489
 490 #
 491 # Stop the server...
 492 #
 493
 494 kill $cupsd
 495
 496 #
 497 # Append the log files for post-mortim...
 498 #
 499
 500 echo "<H1>Log Files</H1>" >>$strfile
 501
 502 echo "<H2>access_log</H2>" >>$strfile
 503 echo "<PRE>" >>$strfile
 504 cat /tmp/$user/log/access_log >>$strfile
 505 echo "</PRE>" >>$strfile
 506
 507 echo "<H2>error_log</H2>" >>$strfile
 508 echo "<PRE>" >>$strfile
 509 cat /tmp/$user/log/error_log >>$strfile
 510 echo "</PRE>" >>$strfile
 511
 512 echo "<H2>page_log</H2>" >>$strfile
 513 echo "<PRE>" >>$strfile
 514 cat /tmp/$user/log/page_log >>$strfile
 515 echo "</PRE>" >>$strfile
 516
 517 if test -f /tmp/$user/log/cupsd.trace; then
 518         echo "<H2>cupsd.trace</H2>" >>$strfile
 519         echo "<PRE>" >>$strfile
 520         cat /tmp/$user/log/cupsd.trace >>$strfile
 521         echo "</PRE>" >>$strfile
 522 fi
 523
 524 #
 525 # Format the reports and tell the user where to find them...
 526 #
 527
 528 echo "Formatting reports..."
 529
 530 cat str-trailer.html >>$strfile
 531
 532 htmlfile=`basename $strfile .shtml`.html
 533 pdffile=`basename $strfile .shtml`.pdf
 534
 535 htmldoc --numbered --verbose --titleimage ../doc/images/cups-large.gif \
 536         -f $htmlfile $strfile
 537 htmldoc --numbered --verbose --titleimage ../doc/images/cups-large.gif \
 538         -f $pdffile $strfile
 539
 540 echo ""
 541
 542 if test $fail != 0; then
 543         echo "$fail tests failed."
 544 else
 545         echo "All tests were successful."
 546 fi
 547
 548 if test "x$valgrind" != x; then
 549         echo "Valgrind log files can be found in /tmp/$user/log."
 550 fi
 551
 552 echo ""
 553 echo "See the following files for details:"
 554 echo ""
 555 echo "    $htmlfile"
 556 echo "    $pdffile"
 557 echo ""
 558
 559 #
 560 # End of "$Id$"
 561 #