test/run-stp-tests.sh

   1 #!/bin/sh
   2 #
   3 # "$Id: run-stp-tests.sh 5091 2006-02-08 18:39:56Z mike $"
   4 #
   5 #   Perform the complete set of IPP compliance tests specified in the
   6 #   CUPS Software Test Plan.
   7 #
   8 #   Copyright 1997-2006 by Easy Software Products, all rights reserved.
   9 #
  10 #   These coded instructions, statements, and computer programs are the
  11 #   property of Easy Software Products and are protected by Federal
  12 #   copyright law.  Distribution and use rights are outlined in the file
  13 #   "LICENSE.txt" which should have been included with this file.  If this
  14 #   file is missing or damaged please contact Easy Software Products
  15 #   at:
  16 #
  17 #       Attn: CUPS Licensing Information
  18 #       Easy Software Products
  19 #       44141 Airport View Drive, Suite 204
  20 #       Hollywood, Maryland 20636 USA
  21 #
  22 #       Voice: (301) 373-9600
  23 #       EMail: cups-info@cups.org
  24 #         WWW: http://www.cups.org
  25 #
  26
  27 #
  28 # Make the IPP test program...
  29 #
  30
  31 make
  32
  33 #
  34 # Figure out the proper echo options...
  35 #
  36
  37 if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null; then
  38         ac_n=-n
  39         ac_c=
  40 else
  41         ac_n=
  42         ac_c='\c'
  43 fi
  44
  45 #
  46 # Greet the tester...
  47 #
  48
  49 echo "Welcome to the CUPS Automated Test Script."
  50 echo ""
  51 echo "Before we begin, it is important that you understand that the larger"
  52 echo "tests require significant amounts of RAM and disk space.  If you"
  53 echo "attempt to run one of the big tests on a system that lacks sufficient"
  54 echo "disk and virtual memory, the UNIX kernel might decide to kill one or"
  55 echo "more system processes that you've grown attached to, like the X"
  56 echo "server.  The question you may want to ask yourself before running a"
  57 echo "large test is: Do you feel lucky?"
  58 echo ""
  59 echo "OK, now that we have the Dirty Harry quote out of the way, please"
  60 echo "choose the type of test you wish to perform:"
  61 echo ""
  62 echo "0 - No testing, keep the scheduler running for me (all systems)"
  63 echo "1 - Basic conformance test, no load testing (all systems)"
  64 echo "2 - Basic conformance test, some load testing (minimum 256MB VM, 50MB disk)"
  65 echo "3 - Basic conformance test, extreme load testing (minimum 1GB VM, 500MB disk)"
  66 echo "4 - Basic conformance test, torture load testing (minimum 2GB VM, 1GB disk)"
  67 echo ""
  68 echo $ac_n "Enter the number of the test you wish to perform: [1] $ac_c"
  69
  70 read testtype
  71 echo ""
  72
  73 case "$testtype" in
  74         0)
  75                 echo "Running in test mode (0)"
  76                 nprinters1=0
  77                 nprinters2=0
  78                 pjobs=0
  79                 ;;
  80         2)
  81                 echo "Running the medium tests (2)"
  82                 nprinters1=10
  83                 nprinters2=20
  84                 pjobs=20
  85                 ;;
  86         3)
  87                 echo "Running the extreme tests (3)"
  88                 nprinters1=500
  89                 nprinters2=1000
  90                 pjobs=100
  91                 ;;
  92         4)
  93                 echo "Running the torture tests (4)"
  94                 nprinters1=10000
  95                 nprinters2=20000
  96                 pjobs=200
  97                 ;;
  98         *)
  99                 echo "Running the timid tests (1)"
 100                 nprinters1=0
 101                 nprinters2=0
 102                 pjobs=0
 103                 ;;
 104 esac
 105
 106 #
 107 # See if we want to do SSL testing...
 108 #
 109
 110 echo ""
 111 echo "Now you can choose whether to create a SSL/TLS encryption key and"
 112 echo "certificate for testing; these tests currently require the OpenSSL"
 113 echo "tools:"
 114 echo ""
 115 echo "0 - Do not do SSL/TLS encryption tests"
 116 echo "1 - Create a SSL/TLS certificate and key, but do not require encryption"
 117 echo "2 - Create a SSL/TLS certificate and key and require encryption"
 118 echo ""
 119 echo $ac_n "Enter the number of the SSL/TLS tests to perform: [0] $ac_c"
 120
 121 read ssltype
 122 echo ""
 123
 124 case "$ssltype" in
 125         1 | 2)
 126                 if test -f server.key; then
 127                         echo "Using existing SSL/TLS certificate and key..."
 128                 else
 129                         echo "Generating SSL/TLS certificate and key..."
 130                         openssl req -new -x509 -keyout server.key -out server.crt -days 3650 -nodes >/dev/null <<EOF
 131 ZZ
 132 Testland
 133 Testville
 134 Test Company
 135
 136 `hostname`
 137 test@domain.com
 138 EOF
 139                 fi
 140                 ;;
 141
 142         *)
 143                 echo "Not using SSL/TLS..."
 144                 ssltype=0
 145                 ;;
 146 esac
 147
 148 #
 149 # Information for the server/tests...
 150 #
 151
 152 user=`whoami`
 153 port=8631
 154 cwd=`pwd`
 155 root=`dirname $cwd`
 156
 157 #
 158 # Make sure that the LPDEST and PRINTER environment variables are
 159 # not included in the environment that is passed to the tests.  These
 160 # will usually cause tests to fail erroneously...
 161 #
 162
 163 typeset +x LPDEST
 164 typeset +x PRINTER
 165
 166 #
 167 # See if we want to use valgrind...
 168 #
 169
 170 echo ""
 171 echo "This test script can use the Valgrind software from:"
 172 echo ""
 173 echo "    http://developer.kde.org/~sewardj/"
 174 echo ""
 175 echo $ac_n "Enter Y to use Valgrind or N to not use Valgrind: [N] $ac_c"
 176
 177 read usevalgrind
 178 echo ""
 179
 180 case "$usevalgrind" in
 181         Y* | y*)
 182                 valgrind="valgrind --tool=memcheck --log-file=/tmp/cups-$user/log/valgrind --error-limit=no --leak-check=yes --trace-children=yes"
 183                 echo "Using Valgrind; log files can be found in /tmp/cups-$user/log..."
 184                 ;;
 185
 186         *)
 187                 valgrind=""
 188                 ;;
 189 esac
 190
 191 #
 192 # Start by creating temporary directories for the tests...
 193 #
 194
 195 rm -rf /tmp/cups-$user
 196 mkdir /tmp/cups-$user
 197 mkdir /tmp/cups-$user/bin
 198 mkdir /tmp/cups-$user/bin/backend
 199 mkdir /tmp/cups-$user/bin/filter
 200 mkdir /tmp/cups-$user/certs
 201 mkdir /tmp/cups-$user/share
 202 mkdir /tmp/cups-$user/share/banners
 203 mkdir /tmp/cups-$user/share/model
 204 mkdir /tmp/cups-$user/interfaces
 205 mkdir /tmp/cups-$user/log
 206 mkdir /tmp/cups-$user/ppd
 207 mkdir /tmp/cups-$user/spool
 208 mkdir /tmp/cups-$user/spool/temp
 209 mkdir /tmp/cups-$user/ssl
 210
 211 ln -s $root/backend/http /tmp/cups-$user/bin/backend
 212 ln -s $root/backend/ipp /tmp/cups-$user/bin/backend
 213 ln -s $root/backend/lpd /tmp/cups-$user/bin/backend
 214 ln -s $root/backend/parallel /tmp/cups-$user/bin/backend
 215 ln -s $root/backend/serial /tmp/cups-$user/bin/backend
 216 ln -s $root/backend/socket /tmp/cups-$user/bin/backend
 217 ln -s $root/backend/usb /tmp/cups-$user/bin/backend
 218 ln -s $root/cgi-bin /tmp/cups-$user/bin
 219 ln -s $root/monitor /tmp/cups-$user/bin
 220 ln -s $root/notifier /tmp/cups-$user/bin
 221 ln -s $root/scheduler /tmp/cups-$user/bin/daemon
 222 ln -s $root/filter/hpgltops /tmp/cups-$user/bin/filter
 223 ln -s $root/filter/imagetops /tmp/cups-$user/bin/filter
 224 ln -s $root/filter/imagetoraster /tmp/cups-$user/bin/filter
 225 ln -s $root/filter/pstops /tmp/cups-$user/bin/filter
 226 ln -s $root/filter/rastertoepson /tmp/cups-$user/bin/filter
 227 ln -s $root/filter/rastertohp /tmp/cups-$user/bin/filter
 228 ln -s $root/filter/texttops /tmp/cups-$user/bin/filter
 229 ln -s $root/pdftops/pdftops /tmp/cups-$user/bin/filter
 230
 231 ln -s $root/data/classified /tmp/cups-$user/share/banners
 232 ln -s $root/data/confidential /tmp/cups-$user/share/banners
 233 ln -s $root/data/secret /tmp/cups-$user/share/banners
 234 ln -s $root/data/standard /tmp/cups-$user/share/banners
 235 ln -s $root/data/topsecret /tmp/cups-$user/share/banners
 236 ln -s $root/data/unclassified /tmp/cups-$user/share/banners
 237 ln -s $root/data /tmp/cups-$user/share/charsets
 238 ln -s $root/data /tmp/cups-$user/share
 239 ln -s $root/fonts /tmp/cups-$user/share
 240 ln -s $root/ppd/*.ppd /tmp/cups-$user/share/model
 241 ln -s $root/templates /tmp/cups-$user/share
 242
 243 if test $ssltype != 0; then
 244         mkdir $root/ssl
 245         cp server.* $root/ssl
 246 fi
 247
 248 #
 249 # Then create the necessary config files...
 250 #
 251
 252 if test $ssltype = 2; then
 253         encryption="Encryption Required"
 254 else
 255         encryption=""
 256 fi
 257
 258 cat >/tmp/cups-$user/cupsd.conf <<EOF
 259 Browsing Off
 260 FileDevice yes
 261 Printcap
 262 Listen 127.0.0.1:$port
 263 User $user
 264 ServerRoot /tmp/cups-$user
 265 StateDir /tmp/cups-$user
 266 ServerBin /tmp/cups-$user/bin
 267 CacheDir /tmp/cups-$user/share
 268 DataDir /tmp/cups-$user/share
 269 FontPath /tmp/cups-$user/share/fonts
 270 DocumentRoot $root/doc
 271 RequestRoot /tmp/cups-$user/spool
 272 TempDir /tmp/cups-$user/spool/temp
 273 MaxLogSize 0
 274 AccessLog /tmp/cups-$user/log/access_log
 275 ErrorLog /tmp/cups-$user/log/error_log
 276 PageLog /tmp/cups-$user/log/page_log
 277 LogLevel debug
 278 PreserveJobHistory Yes
 279 <Policy default>
 280 <Limit All>
 281 Order Deny,Allow
 282 Deny from all
 283 Allow from 127.0.0.1
 284 $encryption
 285 </Limit>
 286 </Policy>
 287 EOF
 288
 289 touch /tmp/cups-$user/classes.conf
 290 touch /tmp/cups-$user/printers.conf
 291
 292 #
 293 # Setup lots of test queues - 500 with PPD files, 500 without...
 294 #
 295
 296 i=1
 297 while test $i -le $nprinters1; do
 298         cat >>/tmp/cups-$user/printers.conf <<EOF
 299 <Printer test-$i>
 300 Accepting Yes
 301 DeviceURI file:/dev/null
 302 Info Test PS printer $i
 303 JobSheets none none
 304 Location CUPS test suite
 305 State Idle
 306 StateMessage Printer $1 is idle.
 307 </Printer>
 308 EOF
 309
 310         cp testps.ppd /tmp/cups-$user/ppd/test-$i.ppd
 311
 312         i=`expr $i + 1`
 313 done
 314
 315 while test $i -le $nprinters2; do
 316         cat >>/tmp/cups-$user/printers.conf <<EOF
 317 <Printer test-$i>
 318 Accepting Yes
 319 DeviceURI file:/dev/null
 320 Info Test raw printer $i
 321 JobSheets none none
 322 Location CUPS test suite
 323 State Idle
 324 StateMessage Printer $1 is idle.
 325 </Printer>
 326 EOF
 327
 328         i=`expr $i + 1`
 329 done
 330
 331 cp /tmp/cups-$user/printers.conf /tmp/cups-$user/printers.conf.orig
 332
 333 cp $root/conf/mime.types /tmp/cups-$user/mime.types
 334 cp $root/conf/mime.convs /tmp/cups-$user/mime.convs
 335
 336 #
 337 # Setup the paths...
 338 #
 339
 340 if test "x$LD_LIBRARY_PATH" = x; then
 341         LD_LIBRARY_PATH="$root/cups:$root/filter"
 342 else
 343         LD_LIBRARY_PATH="$root/cups:$root/filter:$LD_LIBRARY_PATH"
 344 fi
 345
 346 export LD_LIBRARY_PATH
 347
 348 LD_PRELOAD="$root/cups/libcups.so.2:$root/filter/libcupsimage.so.2"
 349 export LD_PRELOAD
 350
 351 if test "x$DYLD_LIBRARY_PATH" = x; then
 352         DYLD_LIBRARY_PATH="$root/cups:$root/filter"
 353 else
 354         DYLD_LIBRARY_PATH="$root/cups:$root/filter:$DYLD_LIBRARY_PATH"
 355 fi
 356
 357 export DYLD_LIBRARY_PATH
 358
 359 if test "x$SHLIB_PATH" = x; then
 360         SHLIB_PATH="$root/cups:$root/filter"
 361 else
 362         SHLIB_PATH="$root/cups:$root/filter:$SHLIB_PATH"
 363 fi
 364
 365 export SHLIB_PATH
 366
 367 CUPS_SERVER=localhost; export CUPS_SERVER
 368 CUPS_SERVERROOT=/tmp/cups-$user; export CUPS_SERVERROOT
 369 CUPS_STATEDIR=/tmp/cups-$user; export CUPS_STATEDIR
 370 CUPS_DATADIR=/tmp/cups-$user/share; export CUPS_DATADIR
 371
 372 #
 373 # Set a new home directory to avoid getting user options mixed in...
 374 #
 375
 376 HOME=/tmp/cups-$user
 377 export HOME
 378
 379 #
 380 # Start the server; run as foreground daemon in the background...
 381 #
 382
 383 echo "Starting scheduler:"
 384 echo "    $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log &"
 385 echo ""
 386
 387 $valgrind ../scheduler/cupsd -c /tmp/cups-$user/cupsd.conf -f >/tmp/cups-$user/log/debug_log &
 388 cupsd=$!
 389
 390 #if test -x /usr/bin/strace; then
 391 #       # Trace system calls in cupsd if we have strace...
 392 #       /usr/bin/strace -tt -o /tmp/cups-$user/log/cupsd.trace -p $cupsd &
 393 #fi
 394
 395 if test "x$testtype" = x0; then
 396         echo "Scheduler is PID $cupsd and is listening on port 8631."
 397         echo ""
 398         echo "Set the IPP_PORT environment variable to 8631 to test the software"
 399         echo "interactively from the command-line."
 400         exit 0
 401 fi
 402
 403 echo "Scheduler is PID $cupsd; run debugger now if you need to."
 404 echo ""
 405 echo $ac_n "Press ENTER to continue... $ac_c"
 406 read junk
 407
 408 IPP_PORT=$port; export IPP_PORT
 409
 410 while true; do
 411         running=`../systemv/lpstat -r 2>/dev/null`
 412         if test "x$running" = "xscheduler is running"; then
 413                 break
 414         fi
 415
 416         echo "Waiting for scheduler to become ready..."
 417         sleep 10
 418 done
 419
 420 #
 421 # Create the test report source file...
 422 #
 423
 424 strfile=cups-str-1.2-`date +%Y-%m-%d`-`whoami`.html
 425
 426 rm -f $strfile
 427 cat str-header.html >$strfile
 428
 429 #
 430 # Run the IPP tests...
 431 #
 432
 433 echo "Running IPP compliance tests..."
 434
 435 echo "<H1>1 - IPP Compliance Tests</H1>" >>$strfile
 436 echo "<P>This section provides the results to the IPP compliance tests" >>$strfile
 437 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 438 echo `date "+%Y-%m-%d"` by `whoami` on `hostname`. >>$strfile
 439 echo "<PRE>" >>$strfile
 440
 441 fail=0
 442 for file in 4*.test; do
 443         echo "Performing $file..."
 444         echo "" >>$strfile
 445
 446         ./ipptest ipp://localhost:$port/printers $file >>$strfile
 447         status=$?
 448
 449         if test $status != 0; then
 450                 echo Test failed.
 451                 fail=`expr $fail + 1`
 452         fi
 453 done
 454
 455 echo "</PRE>" >>$strfile
 456
 457 #
 458 # Run the command tests...
 459 #
 460
 461 echo "Running command tests..."
 462
 463 echo "<H1>2 - Command Tests</H1>" >>$strfile
 464 echo "<P>This section provides the results to the command tests" >>$strfile
 465 echo "outlined in the CUPS Software Test Plan. These tests were run on" >>$strfile
 466 echo `date "+%Y-%m-%d"` by `whoami` on `hostname`. >>$strfile
 467 echo "<PRE>" >>$strfile
 468
 469 for file in 5*.sh; do
 470         echo "Performing $file..."
 471         echo "" >>$strfile
 472         echo "\"$file\":" >>$strfile
 473
 474         sh $file $pjobs >>$strfile
 475         status=$?
 476
 477         if test $status != 0; then
 478                 echo Test failed.
 479                 fail=`expr $fail + 1`
 480         fi
 481 done
 482
 483 echo "</PRE>" >>$strfile
 484
 485 #
 486 # Wait for jobs to complete...
 487 #
 488
 489 while true; do
 490         jobs=`../systemv/lpstat 2>/dev/null`
 491         if test "x$jobs" = "x"; then
 492                 break
 493         fi
 494
 495         echo "Waiting for jobs to complete..."
 496         sleep 10
 497 done
 498
 499 #
 500 # Stop the server...
 501 #
 502
 503 kill $cupsd
 504
 505 #
 506 # Append the log files for post-mortim...
 507 #
 508
 509 echo "<H1>3 - Log Files</H1>" >>$strfile
 510
 511 echo "<H2>access_log</H2>" >>$strfile
 512 echo "<PRE>" >>$strfile
 513 cat /tmp/cups-$user/log/access_log >>$strfile
 514 echo "</PRE>" >>$strfile
 515
 516 echo "<H2>error_log</H2>" >>$strfile
 517 echo "<PRE>" >>$strfile
 518 cat /tmp/cups-$user/log/error_log >>$strfile
 519 echo "</PRE>" >>$strfile
 520
 521 echo "<H2>page_log</H2>" >>$strfile
 522 echo "<PRE>" >>$strfile
 523 cat /tmp/cups-$user/log/page_log >>$strfile
 524 echo "</PRE>" >>$strfile
 525
 526 if test -f /tmp/cups-$user/log/cupsd.trace; then
 527         echo "<H2>cupsd.trace</H2>" >>$strfile
 528         echo "<PRE>" >>$strfile
 529         cat /tmp/cups-$user/log/cupsd.trace >>$strfile
 530         echo "</PRE>" >>$strfile
 531 fi
 532
 533 #
 534 # Format the reports and tell the user where to find them...
 535 #
 536
 537 echo "Formatting reports..."
 538
 539 cat str-trailer.html >>$strfile
 540
 541 echo ""
 542
 543 if test $fail != 0; then
 544         echo "$fail tests failed."
 545 else
 546         echo "All tests were successful."
 547 fi
 548
 549 if test "x$valgrind" != x; then
 550         echo "Valgrind log files can be found in /tmp/cups-$user/log."
 551 fi
 552
 553 echo ""
 554 echo "See the following file for details:"
 555 echo ""
 556 echo "    $strfile"
 557 echo ""
 558
 559 #
 560 # End of "$Id: run-stp-tests.sh 5091 2006-02-08 18:39:56Z mike $"
 561 #