test/run-stp-tests.sh

   1 #!/bin/sh
   2 #
   3 # "$Id: run-stp-tests.sh 5262 2006-03-09 21:26:27Z mike $"
   4 #
   5 #   Perform the complete set of IPP compliance tests specified in the
   6 #   CUPS Software Test Plan.
   7 #
   8 #   Copyright 1997-2006 by Easy Software Products, all rights reserved.
   9 #
  10 #   These coded instructions, statements, and computer programs are the
  11 #   property of Easy Software Products and are protected by Federal
  12 #   copyright law.  Distribution and use rights are outlined in the file
  13 #   "LICENSE.txt" which should have been included with this file.  If this
  14 #   file is missing or damaged please contact Easy Software Products
  15 #   at:
  16 #
  17 #       Attn: CUPS Licensing Information
  18 #       Easy Software Products
  19 #       44141 Airport View Drive, Suite 204
  20 #       Hollywood, Maryland 20636 USA
  21 #
  22 #       Voice: (301) 373-9600
  23 #       EMail: cups-info@cups.org
  24 #         WWW: http://www.cups.org
  25 #
  26
  27 #
  28 # Make the IPP test program...
  29 #
  30
  31 make
  32
  33 #
  34 # Figure out the proper echo options...
  35 #
  36
  37 if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null; then
  38         ac_n=-n
  39         ac_c=
  40 else
  41         ac_n=
  42         ac_c='\c'
  43 fi
  44
  45 #
  46 # Greet the tester...
  47 #
  48
  49 echo "Welcome to the CUPS Automated Test Script."
  50 echo ""
  51 echo "Before we begin, it is important that you understand that the larger"
  52 echo "tests require significant amounts of RAM and disk space.  If you"
  53 echo "attempt to run one of the big tests on a system that lacks sufficient"
  54 echo "disk and virtual memory, the UNIX kernel might decide to kill one or"
  55 echo "more system processes that you've grown attached to, like the X"
  56 echo "server.  The question you may want to ask yourself before running a"
  57 echo "large test is: Do you feel lucky?"
  58 echo ""
  59 echo "OK, now that we have the Dirty Harry quote out of the way, please"
  60 echo "choose the type of test you wish to perform:"
  61 echo ""
  62 echo "0 - No testing, keep the scheduler running for me (all systems)"
  63 echo "1 - Basic conformance test, no load testing (all systems)"
  64 echo "2 - Basic conformance test, some load testing (minimum 256MB VM, 50MB disk)"
  65 echo "3 - Basic conformance test, extreme load testing (minimum 1GB VM, 500MB disk)"
  66 echo "4 - Basic conformance test, torture load testing (minimum 2GB VM, 1GB disk)"
  67 echo ""
  68 echo $ac_n "Enter the number of the test you wish to perform: [1] $ac_c"
  69
  70 read testtype
  71 echo ""
  72
  73 case "$testtype" in
  74         0)
  75                 echo "Running in test mode (0)"
  76                 nprinters1=0
  77                 nprinters2=0
  78                 pjobs=0
  79                 ;;
  80         2)
  81                 echo "Running the medium tests (2)"
  82                 nprinters1=10
  83                 nprinters2=20
  84                 pjobs=20
  85                 ;;
  86         3)
  87                 echo "Running the extreme tests (3)"
  88                 nprinters1=500
  89                 nprinters2=1000
  90                 pjobs=100
  91                 ;;
  92         4)
  93                 echo "Running the torture tests (4)"
  94                 nprinters1=10000
  95                 nprinters2=20000
  96                 pjobs=200
  97                 ;;
  98         *)
  99                 echo "Running the timid tests (1)"
 100                 nprinters1=0
 101                 nprinters2=0
 102                 pjobs=0
 103                 ;;
 104 esac
 105
 106 #
 107 # See if we want to do SSL testing...
 108 #
 109
 110 echo ""
 111 echo "Now you can choose whether to create a SSL/TLS encryption key and"
 112 echo "certificate for testing; these tests currently require the OpenSSL"
 113 echo "tools:"
 114 echo ""
 115 echo "0 - Do not do SSL/TLS encryption tests"
 116 echo "1 - Create a SSL/TLS certificate and key, but do not require encryption"
 117 echo "2 - Create a SSL/TLS certificate and key and require encryption"
 118 echo ""
 119 echo $ac_n "Enter the number of the SSL/TLS tests to perform: [0] $ac_c"
 120
 121 read ssltype
 122 echo ""
 123
 124 case "$ssltype" in
 125         1 | 2)
 126                 if test -f server.key; then
 127                         echo "Using existing SSL/TLS certificate and key..."
 128                 else
 129                         echo "Generating SSL/TLS certificate and key..."
 130                         openssl req -new -x509 -keyout server.key -out server.crt -days 3650 -nodes >/dev/null <<EOF
 131 ZZ
 132 Testland
 133 Testville
 134 Test Company
 135
 136 `hostname`
 137 test@domain.com
 138 EOF
 139                 fi
 140                 ;;
 141
 142         *)
 143                 echo "Not using SSL/TLS..."
 144                 ssltype=0
 145                 ;;
 146 esac
 147
 148 #
 149 # Information for the server/tests...
 150 #
 151
 152 user="$USER"
 153 if test -z "$user"; then
 154         if test -x /usr/ucb/whoami; then
 155                 user=`/usr/ucb/whoami`
 156         else
 157                 user=`whoami`
 158         fi
 159
 160         if test -z "$user"; then
 161                 user="unknown"
 162         fi
 163 fi
 164
 165 port=8631
 166 cwd=`pwd`
 167 root=`dirname $cwd`
 168
 169 #
 170 # Make sure that the LPDEST and PRINTER environment variables are
 171 # not included in the environment that is passed to the tests.  These
 172 # will usually cause tests to fail erroneously...
 173 #
 174
 175 typeset +x LPDEST
 176 typeset +x PRINTER
 177
 178 #
 179 # See if we want to use valgrind...
 180 #
 181
 182 echo ""
 183 echo "This test script can use the Valgrind software from:"
 184 echo ""
 185 echo "    http://developer.kde.org/~sewardj/"
 186 echo ""
 187 echo $ac_n "Enter Y to use Valgrind or N to not use Valgrind: [N] $ac_c"
 188
 189 read usevalgrind
 190 echo ""
 191
 192 case "$usevalgrind" in
 193         Y* | y*)
 194                 valgrind="valgrind --tool=memcheck --log-file=/tmp/cups-$user/log/valgrind --error-limit=no --leak-check=yes --trace-children=yes"
 195                 echo "Using Valgrind; log files can be found in /tmp/cups-$user/log..."
 196                 ;;
 197
 198         *)
 199                 valgrind=""
 200                 ;;
 201 esac
 202
 203 #
 204 # Start by creating temporary directories for the tests...
 205 #
 206
 207 rm -rf /tmp/cups-$user
 208 mkdir /tmp/cups-$user
 209 mkdir /tmp/cups-$user/bin
 210 mkdir /tmp/cups-$user/bin/backend
 211 mkdir /tmp/cups-$user/bin/filter
 212 mkdir /tmp/cups-$user/certs
 213 mkdir /tmp/cups-$user/share
 214 mkdir /tmp/cups-$user/share/banners
 215 mkdir /tmp/cups-$user/share/model
 216 mkdir /tmp/cups-$user/interfaces
 217 mkdir /tmp/cups-$user/log
 218 mkdir /tmp/cups-$user/ppd
 219 mkdir /tmp/cups-$user/spool
 220 mkdir /tmp/cups-$user/spool/temp
 221 mkdir /tmp/cups-$user/ssl
 222
 223 ln -s $root/backend/http /tmp/cups-$user/bin/backend
 224 ln -s $root/backend/ipp /tmp/cups-$user/bin/backend
 225 ln -s $root/backend/lpd /tmp/cups-$user/bin/backend
 226 ln -s $root/backend/parallel /tmp/cups-$user/bin/backend
 227 ln -s $root/backend/serial /tmp/cups-$user/bin/backend
 228 ln -s $root/backend/socket /tmp/cups-$user/bin/backend
 229 ln -s $root/backend/usb /tmp/cups-$user/bin/backend
 230 ln -s $root/cgi-bin /tmp/cups-$user/bin
 231 ln -s $root/monitor /tmp/cups-$user/bin
 232 ln -s $root/notifier /tmp/cups-$user/bin
 233 ln -s $root/scheduler /tmp/cups-$user/bin/daemon
 234 ln -s $root/filter/hpgltops /tmp/cups-$user/bin/filter
 235 ln -s $root/filter/imagetops /tmp/cups-$user/bin/filter
 236 ln -s $root/filter/imagetoraster /tmp/cups-$user/bin/filter
 237 ln -s $root/filter/pstops /tmp/cups-$user/bin/filter
 238 ln -s $root/filter/rastertoepson /tmp/cups-$user/bin/filter
 239 ln -s $root/filter/rastertohp /tmp/cups-$user/bin/filter
 240 ln -s $root/filter/texttops /tmp/cups-$user/bin/filter
 241 ln -s $root/pdftops/pdftops /tmp/cups-$user/bin/filter
 242
 243 ln -s $root/data/classified /tmp/cups-$user/share/banners
 244 ln -s $root/data/confidential /tmp/cups-$user/share/banners
 245 ln -s $root/data/secret /tmp/cups-$user/share/banners
 246 ln -s $root/data/standard /tmp/cups-$user/share/banners
 247 ln -s $root/data/topsecret /tmp/cups-$user/share/banners
 248 ln -s $root/data/unclassified /tmp/cups-$user/share/banners
 249 ln -s $root/data /tmp/cups-$user/share/charsets
 250 ln -s $root/data /tmp/cups-$user/share
 251 ln -s $root/fonts /tmp/cups-$user/share
 252 ln -s $root/ppd/*.ppd /tmp/cups-$user/share/model
 253 ln -s $root/templates /tmp/cups-$user/share
 254
 255 if test $ssltype != 0; then
 256         mkdir $root/ssl
 257         cp server.* $root/ssl
 258 fi
 259
 260 #
 261 # Then create the necessary config files...
 262 #
 263
 264 if test $ssltype = 2; then
 265         encryption="Encryption Required"
 266 else
 267         encryption=""
 268 fi
 269
 270 cat >/tmp/cups-$user/cupsd.conf <<EOF
 271 Browsing Off
 272 FileDevice yes
 273 Printcap
 274 Listen 127.0.0.1:$port
 275 User $user
 276 ServerRoot /tmp/cups-$user
 277 StateDir /tmp/cups-$user
 278 ServerBin /tmp/cups-$user/bin
 279 CacheDir /tmp/cups-$user/share
 280 DataDir /tmp/cups-$user/share
 281 FontPath /tmp/cups-$user/share/fonts
 282 DocumentRoot $root/doc
 283 RequestRoot /tmp/cups-$user/spool
 284 TempDir /tmp/cups-$user/spool/temp
 285 MaxLogSize 0
 286 AccessLog /tmp/cups-$user/log/access_log
 287 ErrorLog /tmp/cups-$user/log/error_log
 288 PageLog /tmp/cups-$user/log/page_log
 289 LogLevel debug
 290 PreserveJobHistory Yes
 291 <Policy default>
 292 <Limit All>
 293 Order Deny,Allow
 294 Deny from all
 295 Allow from 127.0.0.1
 296 $encryption
 297 </Limit>
 298 </Policy>
 299 EOF
 300
 301 touch /tmp/cups-$user/classes.conf
 302 touch /tmp/cups-$user/printers.conf
 303
 304 #
 305 # Setup lots of test queues - 500 with PPD files, 500 without...
 306 #
 307
 308 i=1
 309 while test $i -le $nprinters1; do
 310         cat >>/tmp/cups-$user/printers.conf <<EOF
 311 <Printer test-$i>
 312 Accepting Yes
 313 DeviceURI file:/dev/null
 314 Info Test PS printer $i
 315 JobSheets none none
 316 Location CUPS test suite
 317 State Idle
 318 StateMessage Printer $1 is idle.
 319 </Printer>
 320 EOF
 321
 322         cp testps.ppd /tmp/cups-$user/ppd/test-$i.ppd
 323
 324         i=`expr $i + 1`
 325 done
 326
 327 while test $i -le $nprinters2; do
 328         cat >>/tmp/cups-$user/printers.conf <<EOF
 329 <Printer test-$i>
 330 Accepting Yes
 331 DeviceURI file:/dev/null
 332 Info Test raw printer $i
 333 JobSheets none none
 334 Location CUPS test suite
 335 State Idle
 336 StateMessage Printer $1 is idle.
 337 </Printer>
 338 EOF
 339
 340         i=`expr $i + 1`
 341 done
 342
 343 cp /tmp/cups-$user/printers.conf /tmp/cups-$user/printers.conf.orig
 344
 345 cp $root/conf/mime.types /tmp/cups-$user/mime.types
 346 cp $root/conf/mime.convs /tmp/cups-$user/mime.convs
 347
 348 #
 349 # Setup the paths...
 350 #
 351
 352 if test "x$LD_LIBRARY_PATH" = x; then
 353         LD_LIBRARY_PATH="$root/cups:$root/filter"
 354 else
 355         LD_LIBRARY_PATH="$root/cups:$root/filter:$LD_LIBRARY_PATH"
 356 fi
 357
 358 export LD_LIBRARY_PATH
 359
 360 LD_PRELOAD="$root/cups/libcups.so.2:$root/filter/libcupsimage.so.2"
 361 export LD_PRELOAD
 362
 363 if test "x$DYLD_LIBRARY_PATH" = x; then
 364         DYLD_LIBRARY_PATH="$root/cups:$root/filter"
 365 else
 366         DYLD_LIBRARY_PATH="$root/cups:$root/filter:$DYLD_LIBRARY_PATH"
 367 fi
 368
 369 export DYLD_LIBRARY_PATH
 370
 371 if test "x$SHLIB_PATH" = x; then
 372         SHLIB_PATH="$root/cups:$root/filter"
 373 else
 374         SHLIB_PATH="$root/cups:$root/filter:$SHLIB_PATH"
 375 fi
 376
 377 export SHLIB_PATH
 378
 379 CUPS_SERVER=localhost; export CUPS_SERVER
 380 CUPS_SERVERROOT=/tmp/cups-$user; export CUPS_SERVERROOT
 381 CUPS_STATEDIR=/tmp/cups-$user; export CUPS_STATEDIR
 382 CUPS_DATADIR=/tmp/cups-$user/share; export CUPS_DATADIR
 383
 384 #
 385 # Set a new home directory to avoid getting user options mixed in...
 386 #
 387
 388 HOME=/tmp/cups-$user
 389 export HOME
 390
 391 #
 392 # Start the server; run as foreground daemon in the background...
 393 #
 394
 395 echo "Starting scheduler:"
 396 echo "    $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log &"
 397 echo ""
 398
 399 $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log &
 400 cupsd=$!
 401
 402 #if test -x /usr/bin/strace; then
 403 #       # Trace system calls in cupsd if we have strace...
 404 #       /usr/bin/strace -tt -o /tmp/cups-$user/log/cupsd.trace -p $cupsd &
 405 #fi
 406
 407 if test "x$testtype" = x0; then
 408         echo "Scheduler is PID $cupsd and is listening on port 8631."
 409         echo ""
 410         echo "Set the IPP_PORT environment variable to 8631 to test the software"
 411         echo "interactively from the command-line."
 412         exit 0
 413 fi
 414
 415 echo "Scheduler is PID $cupsd; run debugger now if you need to."
 416 echo ""
 417 echo $ac_n "Press ENTER to continue... $ac_c"
 418 read junk
 419
 420 IPP_PORT=$port; export IPP_PORT
 421
 422 while true; do
 423         running=`../systemv/lpstat -r 2>/dev/null`
 424         if test "x$running" = "xscheduler is running"; then
 425                 break
 426         fi
 427
 428         echo "Waiting for scheduler to become ready..."
 429         sleep 10
 430 done
 431
 432 #
 433 # Create the test report source file...
 434 #
 435
 436 strfile=cups-str-1.2-`date +%Y-%m-%d`-$user.html
 437
 438 rm -f $strfile
 439 cat str-header.html >$strfile
 440
 441 #
 442 # Run the IPP tests...
 443 #
 444
 445 echo "Running IPP compliance tests..."
 446
 447 echo "<H1>1 - IPP Compliance Tests</H1>" >>$strfile
 448 echo "<P>This section provides the results to the IPP compliance tests" >>$strfile
 449 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 450 echo `date "+%Y-%m-%d"` by $user on `hostname`. >>$strfile
 451 echo "<PRE>" >>$strfile
 452
 453 fail=0
 454 for file in 4*.test; do
 455         echo "Performing $file..."
 456         echo "" >>$strfile
 457
 458         ./ipptest ipp://localhost:$port/printers $file >>$strfile
 459         status=$?
 460
 461         if test $status != 0; then
 462                 echo Test failed.
 463                 fail=`expr $fail + 1`
 464         fi
 465 done
 466
 467 echo "</PRE>" >>$strfile
 468
 469 #
 470 # Run the command tests...
 471 #
 472
 473 echo "Running command tests..."
 474
 475 echo "<H1>2 - Command Tests</H1>" >>$strfile
 476 echo "<P>This section provides the results to the command tests" >>$strfile
 477 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 478 echo `date "+%Y-%m-%d"` by $user on `hostname`. >>$strfile
 479 echo "<PRE>" >>$strfile
 480
 481 for file in 5*.sh; do
 482         echo "Performing $file..."
 483         echo "" >>$strfile
 484         echo "\"$file\":" >>$strfile
 485
 486         sh $file $pjobs >>$strfile
 487         status=$?
 488
 489         if test $status != 0; then
 490                 echo Test failed.
 491                 fail=`expr $fail + 1`
 492         fi
 493 done
 494
 495 echo "</PRE>" >>$strfile
 496
 497 #
 498 # Wait for jobs to complete...
 499 #
 500
 501 while true; do
 502         jobs=`../systemv/lpstat 2>/dev/null`
 503         if test "x$jobs" = "x"; then
 504                 break
 505         fi
 506
 507         echo "Waiting for jobs to complete..."
 508         sleep 10
 509 done
 510
 511 #
 512 # Stop the server...
 513 #
 514
 515 kill $cupsd
 516
 517 #
 518 # Append the log files for post-mortim...
 519 #
 520
 521 echo "<H1>3 - Log Files</H1>" >>$strfile
 522
 523 echo "<H2>access_log</H2>" >>$strfile
 524 echo "<PRE>" >>$strfile
 525 cat /tmp/cups-$user/log/access_log >>$strfile
 526 echo "</PRE>" >>$strfile
 527
 528 echo "<H2>error_log</H2>" >>$strfile
 529 echo "<PRE>" >>$strfile
 530 cat /tmp/cups-$user/log/error_log >>$strfile
 531 echo "</PRE>" >>$strfile
 532
 533 echo "<H2>page_log</H2>" >>$strfile
 534 echo "<PRE>" >>$strfile
 535 cat /tmp/cups-$user/log/page_log >>$strfile
 536 echo "</PRE>" >>$strfile
 537
 538 if test -f /tmp/cups-$user/log/cupsd.trace; then
 539         echo "<H2>cupsd.trace</H2>" >>$strfile
 540         echo "<PRE>" >>$strfile
 541         cat /tmp/cups-$user/log/cupsd.trace >>$strfile
 542         echo "</PRE>" >>$strfile
 543 fi
 544
 545 #
 546 # Format the reports and tell the user where to find them...
 547 #
 548
 549 echo "Formatting reports..."
 550
 551 cat str-trailer.html >>$strfile
 552
 553 echo ""
 554
 555 if test $fail != 0; then
 556         echo "$fail tests failed."
 557 else
 558         echo "All tests were successful."
 559 fi
 560
 561 if test "x$valgrind" != x; then
 562         echo "Valgrind log files can be found in /tmp/cups-$user/log."
 563 fi
 564
 565 echo ""
 566 echo "See the following file for details:"
 567 echo ""
 568 echo "    $strfile"
 569 echo ""
 570
 571 #
 572 # End of "$Id: run-stp-tests.sh 5262 2006-03-09 21:26:27Z mike $"
 573 #