test/test-functions

   1 #!/bin/bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # ex: ts=8 sw=4 sts=4 et filetype=sh
   4 PATH=/sbin:/bin:/usr/sbin:/usr/bin
   5 export PATH
   6
   7 LOOKS_LIKE_DEBIAN=$(source /etc/os-release && [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && echo yes || true)
   8 LOOKS_LIKE_ARCH=$(source /etc/os-release && [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && echo yes || true)
   9 LOOKS_LIKE_SUSE=$(source /etc/os-release && [[ " $ID_LIKE " = *" suse "* ]] && echo yes || true)
  10 KERNEL_VER=${KERNEL_VER-$(uname -r)}
  11 KERNEL_MODS="/lib/modules/$KERNEL_VER/"
  12 QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
  13 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
  14 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  15 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  16 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  17 EFI_MOUNT="$(bootctl -p 2>/dev/null || echo /boot)"
  18 QEMU_MEM="${QEMU_MEM:-512M}"
  19
  20 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  21     echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
  22     ROOTLIBDIR=/usr/lib/systemd
  23 fi
  24
  25 PATH_TO_INIT=$ROOTLIBDIR/systemd
  26
  27 BASICTOOLS="test sh bash setsid loadkeys setfont login sulogin gzip sleep echo head tail cat mount umount cryptsetup date dmsetup modprobe sed cmp tee rm true false chmod chown ln xargs"
  28 DEBUGTOOLS="df free ls stty ps ln ip route dmesg dhclient mkdir cp ping dhclient strace less grep id tty touch du sort hostname find vi mv"
  29
  30 STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))"
  31 STATEFILE="$STATEDIR/.testdir"
  32 TESTLOG="$STATEDIR/test.log"
  33
  34 is_built_with_asan() {
  35     if ! type -P objdump >/dev/null; then
  36         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
  37         return 1
  38     fi
  39
  40     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
  41     local _asan_calls=$(objdump -dC $BUILD_DIR/systemd-journald | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
  42     if (( $_asan_calls < 1000 )); then
  43         return 1
  44     else
  45         return 0
  46     fi
  47 }
  48
  49 IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)
  50
  51 if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
  52     STRIP_BINARIES=no
  53     SKIP_INITRD="${SKIP_INITRD:-yes}"
  54     PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
  55     QEMU_MEM="1536M"
  56     QEMU_SMP=4
  57 fi
  58
  59 function find_qemu_bin() {
  60     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
  61     # Either way, only use this version if we aren't running in KVM, because
  62     # nested KVM is flaky still.
  63     if [[ $(systemd-detect-virt -v) != kvm && -z $TEST_NO_KVM ]] ; then
  64         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a kvm qemu-kvm 2>/dev/null | grep '^/' -m1)
  65     fi
  66
  67     [ "$ARCH" ] || ARCH=$(uname -m)
  68     case $ARCH in
  69     x86_64)
  70         # QEMU's own build system calls it qemu-system-x86_64
  71         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-x86_64 2>/dev/null | grep '^/' -m1)
  72         ;;
  73     i*86)
  74         # new i386 version of QEMU
  75         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-i386 2>/dev/null | grep '^/' -m1)
  76
  77         # i386 version of QEMU
  78         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu 2>/dev/null | grep '^/' -m1)
  79         ;;
  80     ppc64*)
  81         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-$ARCH 2>/dev/null | grep '^/' -m1)
  82         ;;
  83     esac
  84
  85     if [ ! -e "$QEMU_BIN" ]; then
  86         echo "Could not find a suitable QEMU binary" >&2
  87         return 1
  88     fi
  89 }
  90
  91 # Return 0 if QEMU did run (then you must check the result state/logs for actual
  92 # success), or 1 if QEMU is not available.
  93 run_qemu() {
  94     if [ -f /etc/machine-id ]; then
  95         read MACHINE_ID < /etc/machine-id
  96         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
  97             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
  98         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
  99             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 100     fi
 101
 102     CONSOLE=ttyS0
 103
 104     if [[ ! "$KERNEL_BIN" ]]; then
 105         if [[ "$LOOKS_LIKE_ARCH" ]]; then
 106             KERNEL_BIN=/boot/vmlinuz-linux
 107         else
 108             [ "$ARCH" ] || ARCH=$(uname -m)
 109             case $ARCH in
 110                 ppc64*)
 111                 KERNEL_BIN=/boot/vmlinux-$KERNEL_VER
 112                 CONSOLE=hvc0
 113                 ;;
 114                 *)
 115                 KERNEL_BIN=/boot/vmlinuz-$KERNEL_VER
 116                 ;;
 117             esac
 118         fi
 119     fi
 120
 121     default_fedora_initrd=/boot/initramfs-${KERNEL_VER}.img
 122     default_debian_initrd=/boot/initrd.img-${KERNEL_VER}
 123     default_arch_initrd=/boot/initramfs-linux-fallback.img
 124     default_suse_initrd=/boot/initrd-${KERNEL_VER}
 125     if [[ ! "$INITRD" ]]; then
 126         if [[ -e "$default_fedora_initrd" ]]; then
 127             INITRD="$default_fedora_initrd"
 128         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 129             INITRD="$default_debian_initrd"
 130         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 131             INITRD="$default_arch_initrd"
 132         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 133             INITRD="$default_suse_initrd"
 134         fi
 135     fi
 136
 137     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 138     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 139     # is not installed or there's some other error when calling it, fall back
 140     # to the original value (QEMU_SMP=1).
 141     if ! [ "$QEMU_SMP" ]; then
 142         if ! QEMU_SMP=$(nproc); then
 143             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 144             QEMU_SMP=1
 145         fi
 146     fi
 147
 148     find_qemu_bin || return 1
 149
 150     local _cgroup_args
 151     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 152         _cgroup_args="systemd.unified_cgroup_hierarchy=yes"
 153     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 154         _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=yes"
 155     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 156         _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=no"
 157     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 158         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 159         exit 1
 160     fi
 161
 162     if [[ "$LOOKS_LIKE_SUSE" ]]; then
 163         PARAMS+="rd.hostonly=0"
 164     elif [[ "$LOOKS_LIKE_ARCH" ]]; then
 165         PARAMS+="rw"
 166     else
 167         PARAMS+="ro"
 168     fi
 169
 170     KERNEL_APPEND="$PARAMS \
 171 root=/dev/sda1 \
 172 raid=noautodetect \
 173 loglevel=2 \
 174 init=$PATH_TO_INIT \
 175 console=$CONSOLE \
 176 selinux=0 \
 177 printk.devkmsg=on \
 178 $_cgroup_args \
 179 $KERNEL_APPEND \
 180 "
 181
 182     QEMU_OPTIONS="-smp $QEMU_SMP \
 183 -net none \
 184 -m $QEMU_MEM \
 185 -nographic \
 186 -kernel $KERNEL_BIN \
 187 -drive format=raw,cache=unsafe,file=${TESTDIR}/rootdisk.img \
 188 "
 189
 190     if [[ "$INITRD" && "$SKIP_INITRD" != "yes" ]]; then
 191         QEMU_OPTIONS="$QEMU_OPTIONS -initrd $INITRD"
 192     fi
 193
 194     # Let's use KVM if it is available, but let's avoid using nested KVM as that is still flaky
 195     if [[ -c /dev/kvm && $(systemd-detect-virt -v) != kvm && -z $TEST_NO_KVM ]] ; then
 196         QEMU_OPTIONS="$QEMU_OPTIONS -machine accel=kvm -enable-kvm -cpu host"
 197     fi
 198
 199     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 200         QEMU_BIN="timeout --foreground $QEMU_TIMEOUT $QEMU_BIN"
 201     fi
 202     (set -x; $QEMU_BIN $QEMU_OPTIONS -append "$KERNEL_APPEND")
 203     rc=$?
 204     if [ "$rc" = 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 205         derror "test timed out after $QEMU_TIMEOUT s"
 206         TIMED_OUT=1
 207     else
 208         [ "$rc" != 0 ] && derror "QEMU failed with exit code $rc"
 209     fi
 210     return 0
 211 }
 212
 213 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 214 # success), or 1 if nspawn is not available.
 215 run_nspawn() {
 216     [[ -d /run/systemd/system ]] || return 1
 217
 218     local _nspawn_cmd="$BUILD_DIR/systemd-nspawn $NSPAWN_ARGUMENTS --register=no --kill-signal=SIGKILL --directory=$TESTDIR/$1 $PATH_TO_INIT $KERNEL_APPEND"
 219     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 220         _nspawn_cmd="timeout --foreground $NSPAWN_TIMEOUT $_nspawn_cmd"
 221     fi
 222
 223     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 224         dwarn "nspawn doesn't support UNIFIED_CGROUP_HIERARCHY=hybrid, skipping"
 225         exit
 226     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 227         _nspawn_cmd="env UNIFIED_CGROUP_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY $_nspawn_cmd"
 228     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 229         _nspawn_cmd="env --unset=UNIFIED_CGROUP_HIERARCHY $_nspawn_cmd"
 230     else
 231         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 232         exit 1
 233     fi
 234
 235     (set -x; $_nspawn_cmd)
 236     rc=$?
 237     if [ "$rc" = 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 238         derror "test timed out after $NSPAWN_TIMEOUT s"
 239         TIMED_OUT=1
 240     else
 241         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 242     fi
 243     return 0
 244 }
 245
 246 setup_basic_environment() {
 247     # create the basic filesystem layout
 248     setup_basic_dirs
 249
 250     install_systemd
 251     install_missing_libraries
 252     install_config_files
 253     create_rc_local
 254     install_basic_tools
 255     install_libnss
 256     install_pam
 257     install_dbus
 258     install_fonts
 259     install_keymaps
 260     install_terminfo
 261     install_execs
 262     install_fsck
 263     install_plymouth
 264     install_debug_tools
 265     install_ld_so_conf
 266     setup_selinux
 267     strip_binaries
 268     install_depmod_files
 269     generate_module_dependencies
 270     if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
 271          create_asan_wrapper
 272     fi
 273 }
 274
 275 setup_selinux() {
 276     # don't forget KERNEL_APPEND='... selinux=1 ...'
 277     if [[ "$SETUP_SELINUX" != "yes" ]]; then
 278         ddebug "Don't setup SELinux"
 279         return 0
 280     fi
 281     ddebug "Setup SELinux"
 282     local _conf_dir=/etc/selinux
 283     local _fixfiles_tools="bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles"
 284
 285     rm -rf $initdir/$_conf_dir
 286     if ! cp -ar $_conf_dir $initdir/$_conf_dir; then
 287         dfatal "Failed to copy $_conf_dir"
 288         exit 1
 289     fi
 290
 291     cat <<EOF >$initdir/etc/systemd/system/autorelabel.service
 292 [Unit]
 293 Description=Relabel all filesystems
 294 DefaultDependencies=no
 295 Requires=local-fs.target
 296 Conflicts=shutdown.target
 297 After=local-fs.target
 298 Before=sysinit.target shutdown.target
 299 ConditionSecurity=selinux
 300 ConditionPathExists=|/.autorelabel
 301
 302 [Service]
 303 ExecStart=/bin/sh -x -c 'echo 0 >/sys/fs/selinux/enforce && fixfiles -f -F relabel && rm /.autorelabel && systemctl --force reboot'
 304 Type=oneshot
 305 TimeoutSec=0
 306 RemainAfterExit=yes
 307 EOF
 308
 309     touch $initdir/.autorelabel
 310     mkdir -p $initdir/etc/systemd/system/basic.target.wants
 311     ln -fs autorelabel.service $initdir/etc/systemd/system/basic.target.wants/autorelabel.service
 312
 313     dracut_install $_fixfiles_tools
 314     dracut_install fixfiles
 315     dracut_install sestatus
 316 }
 317
 318 install_valgrind() {
 319     if ! type -p valgrind; then
 320         dfatal "Failed to install valgrind"
 321         exit 1
 322     fi
 323
 324     local _valgrind_bins=$(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')
 325     dracut_install $_valgrind_bins
 326
 327     local _valgrind_libs=$(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')
 328     dracut_install $_valgrind_libs
 329
 330     local _valgrind_dbg_and_supp=$(
 331         strace -e open valgrind /bin/true 2>&1 >/dev/null |
 332         perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
 333     )
 334     dracut_install $_valgrind_dbg_and_supp
 335 }
 336
 337 create_valgrind_wrapper() {
 338     local _valgrind_wrapper=$initdir/$ROOTLIBDIR/systemd-under-valgrind
 339     ddebug "Create $_valgrind_wrapper"
 340     cat >$_valgrind_wrapper <<EOF
 341 #!/bin/bash
 342
 343 mount -t proc proc /proc
 344 exec valgrind --leak-check=full --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 345 EOF
 346     chmod 0755 $_valgrind_wrapper
 347 }
 348
 349 create_asan_wrapper() {
 350     local _asan_wrapper=$initdir/$ROOTLIBDIR/systemd-under-asan
 351     ddebug "Create $_asan_wrapper"
 352     cat >$_asan_wrapper <<EOF
 353 #!/bin/bash
 354
 355 set -x
 356
 357 DEFAULT_ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
 358 DEFAULT_UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
 359 DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS"
 360
 361 mount -t proc proc /proc
 362 mount -t sysfs sysfs /sys
 363 mount -o remount,rw /
 364
 365 PATH_TO_ASAN=\$(find / -name '*libasan*' | sed 1q)
 366 if [[ "\$PATH_TO_ASAN" ]]; then
 367   # A lot of services (most notably dbus) won't start without preloading libasan
 368   # See https://github.com/systemd/systemd/issues/5004
 369   DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=\$PATH_TO_ASAN"
 370 fi
 371 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
 372 echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 373
 374 # ASAN and syscall filters aren't compatible with each other.
 375 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
 376
 377 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
 378 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
 379 JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 380 mkdir -p "\$JOURNALD_CONF_DIR"
 381 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 382
 383 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
 384 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
 385 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
 386 printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"
 387
 388 # 90s isn't enough for some services to finish when literally everything is run
 389 # under ASan+UBSan in containers, which, in turn, are run in VMs.
 390 # Let's limit which environments such services should be executed in.
 391 mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
 392 printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
 393
 394 # Let's override another hard-coded timeout that kicks in too early
 395 mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
 396 printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf
 397
 398 # The 'mount' utility doesn't behave well under libasan, causing unexpected
 399 # fails during boot and subsequent test results check:
 400 # bash-5.0# mount -o remount,rw -v /
 401 # mount: /dev/sda1 mounted on /.
 402 # bash-5.0# echo \$?
 403 # 1
 404 # Let's workaround this by clearing the previously set LD_PRELOAD env variable,
 405 # so the libasan library is not loaded for this particular service
 406 REMOUNTFS_CONF_DIR=/etc/systemd/system/systemd-remount-fs.service.d
 407 mkdir -p "\$REMOUNTFS_CONF_DIR"
 408 printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$REMOUNTFS_CONF_DIR/env.conf"
 409
 410 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 411 exec  $ROOTLIBDIR/systemd "\$@"
 412 EOF
 413
 414     chmod 0755 $_asan_wrapper
 415 }
 416
 417 create_strace_wrapper() {
 418     local _strace_wrapper=$initdir/$ROOTLIBDIR/systemd-under-strace
 419     ddebug "Create $_strace_wrapper"
 420     cat >$_strace_wrapper <<EOF
 421 #!/bin/bash
 422
 423 exec strace -D -o /strace.out $ROOTLIBDIR/systemd "\$@"
 424 EOF
 425     chmod 0755 $_strace_wrapper
 426 }
 427
 428 install_fsck() {
 429     dracut_install /sbin/fsck*
 430     dracut_install -o /bin/fsck*
 431
 432     # fskc.reiserfs calls reiserfsck. so, install it
 433     dracut_install -o reiserfsck
 434 }
 435
 436 install_dmevent() {
 437     instmods dm_crypt =crypto
 438     type -P dmeventd >/dev/null && dracut_install dmeventd
 439     inst_libdir_file "libdevmapper-event.so*"
 440     if [[ "$LOOKS_LIKE_DEBIAN" ]]; then
 441         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
 442         # and since buster/bionic 95-dm-notify.rules
 443         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
 444         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
 445     else
 446         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
 447     fi
 448 }
 449
 450 install_systemd() {
 451     # install compiled files
 452     local _ninja_bin=$(type -P ninja || type -P ninja-build)
 453     if [[ -z "$_ninja_bin" ]]; then
 454         dfatal "ninja was not found"
 455         exit 1
 456     fi
 457     (set -x; DESTDIR=$initdir "$_ninja_bin" -C $BUILD_DIR install)
 458     # remove unneeded documentation
 459     rm -fr $initdir/usr/share/{man,doc}
 460     # we strip binaries since debug symbols increase binaries size a lot
 461     # and it could fill the available space
 462     strip_binaries
 463
 464    [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse
 465
 466     # enable debug logging in PID1
 467     echo LogLevel=debug >> $initdir/etc/systemd/system.conf
 468 }
 469
 470 get_ldpath() {
 471     local _bin="$1"
 472     objdump -p "$_bin" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :
 473 }
 474
 475 install_missing_libraries() {
 476     # install possible missing libraries
 477     for i in $initdir{,/usr}/{sbin,bin}/* $initdir{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
 478         LD_LIBRARY_PATH=$(get_ldpath $i) inst_libs $i
 479     done
 480 }
 481
 482 create_empty_image() {
 483     local _size=500
 484     if [[ "$STRIP_BINARIES" = "no" ]]; then
 485         _size=$((2*_size))
 486     fi
 487     rm -f "$TESTDIR/rootdisk.img"
 488     # Create the blank file to use as a root filesystem
 489     dd if=/dev/null of="$TESTDIR/rootdisk.img" bs=1M seek="$_size"
 490     LOOPDEV=$(losetup --show -P -f $TESTDIR/rootdisk.img)
 491     [ -b "$LOOPDEV" ] || return 1
 492     echo "LOOPDEV=$LOOPDEV" >> $STATEFILE
 493     sfdisk "$LOOPDEV" <<EOF
 494 ,$((_size-50))M
 495 ,
 496 EOF
 497
 498     udevadm settle
 499
 500     local _label="-L systemd"
 501     # mkfs.reiserfs doesn't know -L. so, use --label instead
 502     [[ "$FSTYPE" == "reiserfs" ]] && _label="--label systemd"
 503     if ! mkfs -t "${FSTYPE}" ${_label} "${LOOPDEV}p1" -q; then
 504         dfatal "Failed to mkfs -t ${FSTYPE}"
 505         exit 1
 506     fi
 507 }
 508
 509 check_asan_reports() {
 510     local ret=0
 511     local root="$1"
 512
 513     if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
 514         ls -l "$root"
 515         if [[ -e "$root/systemd.asan.log.1" ]]; then
 516             cat "$root/systemd.asan.log.1"
 517             ret=$(($ret+1))
 518         fi
 519
 520         journald_report=$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)
 521         if [[ ! -z "$journald_report" ]]; then
 522             printf "%s\n" "$journald_report"
 523             cat "$root/systemd-journald.out" || true
 524             ret=$(($ret+1))
 525         fi
 526
 527         pids=$(
 528             "$BUILD_DIR/journalctl" -D "$root/var/log/journal" | perl -alne '
 529                  BEGIN {
 530                      %services_to_ignore = (
 531                          "dbus-daemon" => undef,
 532                      );
 533                  }
 534                  print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
 535         )
 536         if [[ ! -z "$pids" ]]; then
 537             ret=$(($ret+1))
 538             for pid in $pids; do
 539                 "$BUILD_DIR/journalctl" -D "$root/var/log/journal" _PID=$pid --no-pager
 540             done
 541         fi
 542     fi
 543
 544     return $ret
 545 }
 546
 547 check_result_nspawn() {
 548     local ret=1
 549     local journald_report=""
 550     local pids=""
 551     [[ -e $TESTDIR/$1/testok ]] && ret=0
 552     [[ -f $TESTDIR/$1/failed ]] && cp -a $TESTDIR/$1/failed $TESTDIR
 553     cp -a $TESTDIR/$1/var/log/journal $TESTDIR
 554     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
 555     ls -l $TESTDIR/journal/*/*.journal
 556     test -s $TESTDIR/failed && ret=$(($ret+1))
 557     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
 558     check_asan_reports "$TESTDIR/$1" || ret=$(($ret+1))
 559     return $ret
 560 }
 561
 562 # can be overridden in specific test
 563 check_result_qemu() {
 564     local ret=1
 565     mkdir -p $TESTDIR/root
 566     mount ${LOOPDEV}p1 $TESTDIR/root
 567     [[ -e $TESTDIR/root/testok ]] && ret=0
 568     [[ -f $TESTDIR/root/failed ]] && cp -a $TESTDIR/root/failed $TESTDIR
 569     cp -a $TESTDIR/root/var/log/journal $TESTDIR
 570     check_asan_reports "$TESTDIR/root" || ret=$(($ret+1))
 571     umount $TESTDIR/root
 572     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
 573     ls -l $TESTDIR/journal/*/*.journal
 574     test -s $TESTDIR/failed && ret=$(($ret+1))
 575     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
 576     return $ret
 577 }
 578
 579 strip_binaries() {
 580     if [[ "$STRIP_BINARIES" = "no" ]]; then
 581         ddebug "Don't strip binaries"
 582         return 0
 583     fi
 584     ddebug "Strip binaries"
 585     find "$initdir" -executable -not -path '*/lib/modules/*.ko' -type f | \
 586         xargs strip --strip-unneeded |& \
 587         grep -v 'file format not recognized' | \
 588         ddebug
 589 }
 590
 591 create_rc_local() {
 592     mkdir -p $initdir/etc/rc.d
 593     cat >$initdir/etc/rc.d/rc.local <<EOF
 594 #!/bin/bash
 595 exit 0
 596 EOF
 597     chmod 0755 $initdir/etc/rc.d/rc.local
 598 }
 599
 600 install_execs() {
 601     ddebug "install any Execs from the service files"
 602     (
 603     export PKG_CONFIG_PATH=$BUILD_DIR/src/core/
 604     systemdsystemunitdir=$(pkg-config --variable=systemdsystemunitdir systemd)
 605     systemduserunitdir=$(pkg-config --variable=systemduserunitdir systemd)
 606     sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' $initdir/{$systemdsystemunitdir,$systemduserunitdir}/*.service \
 607          | sort -u | while read i; do
 608          # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
 609          # also, plymouth is pulled in by rescue.service, but even there the exit code
 610          # is ignored; as it's not present on some distros, don't fail if it doesn't exist
 611          inst $i || [ "${i%.local}" != "$i" ] || [ "${i%systemd-update-done}" != "$i" ] || [ "/bin/plymouth" == "$i" ]
 612      done
 613     )
 614 }
 615
 616 generate_module_dependencies() {
 617     if [[ -d $initdir/lib/modules/$KERNEL_VER ]] && \
 618         ! depmod -a -b "$initdir" $KERNEL_VER; then
 619             dfatal "\"depmod -a $KERNEL_VER\" failed."
 620             exit 1
 621     fi
 622 }
 623
 624 install_depmod_files() {
 625     inst /lib/modules/$KERNEL_VER/modules.order
 626     inst /lib/modules/$KERNEL_VER/modules.builtin
 627 }
 628
 629 install_plymouth() {
 630     # install plymouth, if found... else remove plymouth service files
 631     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
 632     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
 633     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
 634     #         dracut_install plymouth plymouthd
 635     # else
 636         rm -f $initdir/{usr/lib,etc}/systemd/system/plymouth* $initdir/{usr/lib,etc}/systemd/system/*/plymouth*
 637     # fi
 638 }
 639
 640 install_ld_so_conf() {
 641     cp -a /etc/ld.so.conf* $initdir/etc
 642     ldconfig -r "$initdir"
 643 }
 644
 645 install_config_files() {
 646     inst /etc/sysconfig/init || true
 647     inst /etc/passwd
 648     inst /etc/shadow
 649     inst /etc/login.defs
 650     inst /etc/group
 651     inst /etc/shells
 652     inst /etc/nsswitch.conf
 653     inst /etc/pam.conf || true
 654     inst /etc/securetty || true
 655     inst /etc/os-release
 656     inst /etc/localtime
 657     # we want an empty environment
 658     > $initdir/etc/environment
 659     > $initdir/etc/machine-id
 660     # set the hostname
 661     echo systemd-testsuite > $initdir/etc/hostname
 662     # fstab
 663     if [[ "$LOOKS_LIKE_SUSE" ]]; then
 664        ROOTMOUNT="/dev/sda1           /       ${FSTYPE}    rw 0 1"
 665     else
 666        ROOTMOUNT="LABEL=systemd           /       ${FSTYPE}    rw 0 1"
 667     fi
 668
 669     cat >$initdir/etc/fstab <<EOF
 670 $ROOTMOUNT
 671 EOF
 672 }
 673
 674 install_basic_tools() {
 675     [[ $BASICTOOLS ]] && dracut_install $BASICTOOLS
 676     dracut_install -o sushell
 677     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
 678     dracut_install -o ldconfig.real
 679 }
 680
 681 install_debug_tools() {
 682     [[ $DEBUGTOOLS ]] && dracut_install $DEBUGTOOLS
 683
 684     if [[ $INTERACTIVE_DEBUG ]]; then
 685         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
 686         local _getty_override="$initdir/etc/systemd/system/serial-getty@.service.d"
 687         mkdir -p "$_getty_override"
 688         echo -e "[Service]\nEnvironment=TERM=linux" > "$_getty_override/default-TERM.conf"
 689
 690         cat > "$initdir/etc/motd" << EOF
 691 To adjust the terminal size use:
 692     export COLUMNS=xx
 693     export LINES=yy
 694 or
 695     stty cols xx rows yy
 696 EOF
 697     fi
 698 }
 699
 700 install_libnss() {
 701     # install libnss_files for login
 702     NSS_LIBS=$(LD_DEBUG=files getent passwd 2>&1 >/dev/null |sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
 703     dracut_install $NSS_LIBS
 704 }
 705
 706 install_dbus() {
 707     inst $ROOTLIBDIR/system/dbus.socket
 708
 709     # Fedora rawhide replaced dbus.service with dbus-daemon.service
 710     if [ -f $ROOTLIBDIR/system/dbus-daemon.service ]; then
 711         inst $ROOTLIBDIR/system/dbus-daemon.service
 712         # Alias symlink
 713         inst_symlink /etc/systemd/system/dbus.service
 714     else
 715         inst $ROOTLIBDIR/system/dbus.service
 716     fi
 717     # Newer Fedora versions use dbus-broker by default. Let's install it is available.
 718     [ -f /usr/bin/dbus-broker ] && inst /usr/bin/dbus-broker
 719     [ -f /usr/bin/dbus-broker-launch ] && inst /usr/bin/dbus-broker-launch
 720
 721     find \
 722         /etc/dbus-1 /usr/share/dbus-1 -xtype f \
 723         | while read file; do
 724         inst $file
 725     done
 726 }
 727
 728 install_pam() {
 729     (
 730     if [[ "$LOOKS_LIKE_DEBIAN" ]] && type -p dpkg-architecture &>/dev/null; then
 731         find "/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security" -xtype f
 732     else
 733         find /lib*/security -xtype f
 734     fi
 735     find /etc/pam.d /etc/security -xtype f
 736     ) | while read file; do
 737         inst $file
 738     done
 739
 740     # pam_unix depends on unix_chkpwd.
 741     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
 742     dracut_install -o unix_chkpwd
 743
 744     [[ "$LOOKS_LIKE_DEBIAN" ]] &&
 745         cp /etc/pam.d/systemd-user $initdir/etc/pam.d/
 746
 747     # set empty root password for easy debugging
 748     sed -i 's/^root:x:/root::/' $initdir/etc/passwd
 749 }
 750
 751 install_keymaps() {
 752     # The first three paths may be deprecated.
 753     # It seems now the last two paths are used by many distributions.
 754     for i in \
 755         /usr/lib/kbd/keymaps/include/* \
 756         /usr/lib/kbd/keymaps/i386/include/* \
 757         /usr/lib/kbd/keymaps/i386/qwerty/us.* \
 758         /usr/lib/kbd/keymaps/legacy/include/* \
 759         /usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
 760             [[ -f $i ]] || continue
 761             inst $i
 762     done
 763
 764     # When it takes any argument, then install more keymaps.
 765     if [[ -n $1 ]]; then
 766         for i in \
 767         /usr/lib/kbd/keymaps/i386/*/* \
 768         /usr/lib/kbd/keymaps/legacy/i386/*/*; do
 769             [[ -f $i ]] || continue
 770             inst $i
 771         done
 772     fi
 773 }
 774
 775 install_zoneinfo() {
 776     for i in /usr/share/zoneinfo/{,*/,*/*/}*; do
 777         [[ -f $i ]] || continue
 778         inst $i
 779     done
 780 }
 781
 782 install_fonts() {
 783     for i in \
 784         /usr/lib/kbd/consolefonts/eurlatgr* \
 785         /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
 786             [[ -f $i ]] || continue
 787             inst $i
 788     done
 789 }
 790
 791 install_terminfo() {
 792     for _terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
 793         [ -f ${_terminfodir}/l/linux ] && break
 794     done
 795     dracut_install -o ${_terminfodir}/l/linux
 796 }
 797
 798 setup_testsuite() {
 799     cp $TEST_BASE_DIR/testsuite.target $initdir/etc/systemd/system/
 800     cp $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/
 801
 802     mkdir -p $initdir/etc/systemd/system/testsuite.target.wants
 803     ln -fs $TEST_BASE_DIR/testsuite.service $initdir/etc/systemd/system/testsuite.target.wants/testsuite.service
 804     # Don't shutdown the machine after running the test when INTERACTIVE_DEBUG is set
 805     [[ -z $INTERACTIVE_DEBUG ]] && ln -fs $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/testsuite.target.wants/end.service
 806
 807     # make the testsuite the default target
 808     ln -fs testsuite.target $initdir/etc/systemd/system/default.target
 809 }
 810
 811 setup_nspawn_root() {
 812     rm -fr $TESTDIR/nspawn-root
 813     ddebug "cp -ar $initdir $TESTDIR/nspawn-root"
 814     cp -ar $initdir $TESTDIR/nspawn-root
 815     # we don't mount in the nspawn root
 816     rm -f $TESTDIR/nspawn-root/etc/fstab
 817     if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
 818         cp -ar $TESTDIR/nspawn-root $TESTDIR/unprivileged-nspawn-root
 819     fi
 820 }
 821
 822 setup_basic_dirs() {
 823     mkdir -p $initdir/run
 824     mkdir -p $initdir/etc/systemd/system
 825     mkdir -p $initdir/var/log/journal
 826
 827     for d in usr/bin usr/sbin bin etc lib "$libdir" sbin tmp usr var var/log dev proc sys sysroot root run run/lock run/initramfs; do
 828         if [ -L "/$d" ]; then
 829             inst_symlink "/$d"
 830         else
 831             inst_dir "/$d"
 832         fi
 833     done
 834
 835     ln -sfn /run "$initdir/var/run"
 836     ln -sfn /run/lock "$initdir/var/lock"
 837 }
 838
 839 inst_libs() {
 840     local _bin=$1
 841     local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
 842     local _file _line
 843
 844     LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
 845         [[ $_line = 'not a dynamic executable' ]] && break
 846
 847         if [[ $_line =~ $_so_regex ]]; then
 848             _file=${BASH_REMATCH[1]}
 849             [[ -e ${initdir}/$_file ]] && continue
 850             inst_library "$_file"
 851             continue
 852         fi
 853
 854         if [[ $_line =~ not\ found ]]; then
 855             dfatal "Missing a shared library required by $_bin."
 856             dfatal "Run \"ldd $_bin\" to find out what it is."
 857             dfatal "$_line"
 858             dfatal "dracut cannot create an initrd."
 859             exit 1
 860         fi
 861     done
 862 }
 863
 864 import_testdir() {
 865     [[ -e $STATEFILE ]] && . $STATEFILE
 866     if [[ ! -d "$TESTDIR" ]]; then
 867         if [[ -z "$TESTDIR" ]]; then
 868             TESTDIR=$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)
 869         else
 870             mkdir -p "$TESTDIR"
 871         fi
 872
 873         echo "TESTDIR=\"$TESTDIR\"" > $STATEFILE
 874         export TESTDIR
 875     fi
 876 }
 877
 878 import_initdir() {
 879     initdir=$TESTDIR/root
 880     export initdir
 881 }
 882
 883 ## @brief Converts numeric logging level to the first letter of level name.
 884 #
 885 # @param lvl Numeric logging level in range from 1 to 6.
 886 # @retval 1 if @a lvl is out of range.
 887 # @retval 0 if @a lvl is correct.
 888 # @result Echoes first letter of level name.
 889 _lvl2char() {
 890     case "$1" in
 891         1) echo F;;
 892         2) echo E;;
 893         3) echo W;;
 894         4) echo I;;
 895         5) echo D;;
 896         6) echo T;;
 897         *) return 1;;
 898     esac
 899 }
 900
 901 ## @brief Internal helper function for _do_dlog()
 902 #
 903 # @param lvl Numeric logging level.
 904 # @param msg Message.
 905 # @retval 0 It's always returned, even if logging failed.
 906 #
 907 # @note This function is not supposed to be called manually. Please use
 908 # dtrace(), ddebug(), or others instead which wrap this one.
 909 #
 910 # This function calls _do_dlog() either with parameter msg, or if
 911 # none is given, it will read standard input and will use every line as
 912 # a message.
 913 #
 914 # This enables:
 915 # dwarn "This is a warning"
 916 # echo "This is a warning" | dwarn
 917 LOG_LEVEL=${LOG_LEVEL:-4}
 918
 919 dlog() {
 920     [ -z "$LOG_LEVEL" ] && return 0
 921     [ $1 -le $LOG_LEVEL ] || return 0
 922     local lvl="$1"; shift
 923     local lvlc=$(_lvl2char "$lvl") || return 0
 924
 925     if [ $# -ge 1 ]; then
 926         echo "$lvlc: $*"
 927     else
 928         while read line; do
 929             echo "$lvlc: " "$line"
 930         done
 931     fi
 932 }
 933
 934 ## @brief Logs message at TRACE level (6)
 935 #
 936 # @param msg Message.
 937 # @retval 0 It's always returned, even if logging failed.
 938 dtrace() {
 939     set +x
 940     dlog 6 "$@"
 941     [ -n "$debug" ] && set -x || :
 942 }
 943
 944 ## @brief Logs message at DEBUG level (5)
 945 #
 946 # @param msg Message.
 947 # @retval 0 It's always returned, even if logging failed.
 948 ddebug() {
 949 #    set +x
 950     dlog 5 "$@"
 951 #    [ -n "$debug" ] && set -x || :
 952 }
 953
 954 ## @brief Logs message at INFO level (4)
 955 #
 956 # @param msg Message.
 957 # @retval 0 It's always returned, even if logging failed.
 958 dinfo() {
 959     set +x
 960     dlog 4 "$@"
 961     [ -n "$debug" ] && set -x || :
 962 }
 963
 964 ## @brief Logs message at WARN level (3)
 965 #
 966 # @param msg Message.
 967 # @retval 0 It's always returned, even if logging failed.
 968 dwarn() {
 969     set +x
 970     dlog 3 "$@"
 971     [ -n "$debug" ] && set -x || :
 972 }
 973
 974 ## @brief Logs message at ERROR level (2)
 975 #
 976 # @param msg Message.
 977 # @retval 0 It's always returned, even if logging failed.
 978 derror() {
 979 #    set +x
 980     dlog 2 "$@"
 981 #    [ -n "$debug" ] && set -x || :
 982 }
 983
 984 ## @brief Logs message at FATAL level (1)
 985 #
 986 # @param msg Message.
 987 # @retval 0 It's always returned, even if logging failed.
 988 dfatal() {
 989     set +x
 990     dlog 1 "$@"
 991     [ -n "$debug" ] && set -x || :
 992 }
 993
 994
 995 # Generic substring function.  If $2 is in $1, return 0.
 996 strstr() { [ "${1#*$2*}" != "$1" ]; }
 997
 998 # normalize_path <path>
 999 # Prints the normalized path, where it removes any duplicated
1000 # and trailing slashes.
1001 # Example:
1002 # $ normalize_path ///test/test//
1003 # /test/test
1004 normalize_path() {
1005     shopt -q -s extglob
1006     set -- "${1//+(\/)//}"
1007     shopt -q -u extglob
1008     echo "${1%/}"
1009 }
1010
1011 # convert_abs_rel <from> <to>
1012 # Prints the relative path, when creating a symlink to <to> from <from>.
1013 # Example:
1014 # $ convert_abs_rel /usr/bin/test /bin/test-2
1015 # ../../bin/test-2
1016 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
1017 convert_abs_rel() {
1018     local __current __absolute __abssize __cursize __newpath
1019     local -i __i __level
1020
1021     set -- "$(normalize_path "$1")" "$(normalize_path "$2")"
1022
1023     # corner case #1 - self looping link
1024     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
1025
1026     # corner case #2 - own dir link
1027     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
1028
1029     IFS="/" __current=($1)
1030     IFS="/" __absolute=($2)
1031
1032     __abssize=${#__absolute[@]}
1033     __cursize=${#__current[@]}
1034
1035     while [[ ${__absolute[__level]} == ${__current[__level]} ]]
1036     do
1037         (( __level++ ))
1038         if (( __level > __abssize || __level > __cursize ))
1039         then
1040             break
1041         fi
1042     done
1043
1044     for ((__i = __level; __i < __cursize-1; __i++))
1045     do
1046         if ((__i > __level))
1047         then
1048             __newpath=$__newpath"/"
1049         fi
1050         __newpath=$__newpath".."
1051     done
1052
1053     for ((__i = __level; __i < __abssize; __i++))
1054     do
1055         if [[ -n $__newpath ]]
1056         then
1057             __newpath=$__newpath"/"
1058         fi
1059         __newpath=$__newpath${__absolute[__i]}
1060     done
1061
1062     echo "$__newpath"
1063 }
1064
1065
1066 # Install a directory, keeping symlinks as on the original system.
1067 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
1068 # will create ${initdir}/lib64, ${initdir}/lib64/file,
1069 # and a symlink ${initdir}/lib -> lib64.
1070 inst_dir() {
1071     [[ -e ${initdir}/"$1" ]] && return 0  # already there
1072
1073     local _dir="$1" _part="${1%/*}" _file
1074     while [[ "$_part" != "${_part%/*}" ]] && ! [[ -e "${initdir}/${_part}" ]]; do
1075         _dir="$_part $_dir"
1076         _part=${_part%/*}
1077     done
1078
1079     # iterate over parent directories
1080     for _file in $_dir; do
1081         [[ -e "${initdir}/$_file" ]] && continue
1082         if [[ -L $_file ]]; then
1083             inst_symlink "$_file"
1084         else
1085             # create directory
1086             mkdir -m 0755 -p "${initdir}/$_file" || return 1
1087             [[ -e "$_file" ]] && chmod --reference="$_file" "${initdir}/$_file"
1088             chmod u+w "${initdir}/$_file"
1089         fi
1090     done
1091 }
1092
1093 # $1 = file to copy to ramdisk
1094 # $2 (optional) Name for the file on the ramdisk
1095 # Location of the image dir is assumed to be $initdir
1096 # We never overwrite the target if it exists.
1097 inst_simple() {
1098     [[ -f "$1" ]] || return 1
1099     strstr "$1" "/" || return 1
1100
1101     local _src=$1 target="${2:-$1}"
1102     if ! [[ -d ${initdir}/$target ]]; then
1103         [[ -e ${initdir}/$target ]] && return 0
1104         [[ -L ${initdir}/$target ]] && return 0
1105         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
1106     fi
1107     # install checksum files also
1108     if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
1109         inst "${_src%/*}/.${_src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
1110     fi
1111     ddebug "Installing $_src"
1112     cp --sparse=always -pfL "$_src" "${initdir}/$target"
1113 }
1114
1115 # find symlinks linked to given library file
1116 # $1 = library file
1117 # Function searches for symlinks by stripping version numbers appended to
1118 # library filename, checks if it points to the same target and finally
1119 # prints the list of symlinks to stdout.
1120 #
1121 # Example:
1122 # rev_lib_symlinks libfoo.so.8.1
1123 # output: libfoo.so.8 libfoo.so
1124 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
1125 rev_lib_symlinks() {
1126     [[ ! $1 ]] && return 0
1127
1128     local fn="$1" orig="$(readlink -f "$1")" links=''
1129
1130     [[ ${fn} =~ .*\.so\..* ]] || return 1
1131
1132     until [[ ${fn##*.} == so ]]; do
1133         fn="${fn%.*}"
1134         [[ -L ${fn} && $(readlink -f "${fn}") == ${orig} ]] && links+=" ${fn}"
1135     done
1136
1137     echo "${links}"
1138 }
1139
1140 # Same as above, but specialized to handle dynamic libraries.
1141 # It handles making symlinks according to how the original library
1142 # is referenced.
1143 inst_library() {
1144     local _src="$1" _dest=${2:-$1} _lib _reallib _symlink
1145     strstr "$1" "/" || return 1
1146     [[ -e $initdir/$_dest ]] && return 0
1147     if [[ -L $_src ]]; then
1148         # install checksum files also
1149         if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
1150             inst "${_src%/*}/.${_src##*/}.hmac" "${_dest%/*}/.${_dest##*/}.hmac"
1151         fi
1152         _reallib=$(readlink -f "$_src")
1153         inst_simple "$_reallib" "$_reallib"
1154         inst_dir "${_dest%/*}"
1155         [[ -d "${_dest%/*}" ]] && _dest=$(readlink -f "${_dest%/*}")/${_dest##*/}
1156         ln -sfn $(convert_abs_rel "${_dest}" "${_reallib}") "${initdir}/${_dest}"
1157     else
1158         inst_simple "$_src" "$_dest"
1159     fi
1160
1161     # Create additional symlinks.  See rev_symlinks description.
1162     for _symlink in $(rev_lib_symlinks $_src) $(rev_lib_symlinks $_reallib); do
1163         [[ -e $initdir/$_symlink ]] || {
1164             ddebug "Creating extra symlink: $_symlink"
1165             inst_symlink $_symlink
1166         }
1167     done
1168 }
1169
1170 # find a binary.  If we were not passed the full path directly,
1171 # search in the usual places to find the binary.
1172 find_binary() {
1173     if [[ -z ${1##/*} ]]; then
1174         if [[ -x $1 ]] || { strstr "$1" ".so" && ldd $1 &>/dev/null; };  then
1175             echo $1
1176             return 0
1177         fi
1178     fi
1179
1180     type -P $1
1181 }
1182
1183 # Same as above, but specialized to install binary executables.
1184 # Install binary executable, and all shared library dependencies, if any.
1185 inst_binary() {
1186     local _bin _target
1187     _bin=$(find_binary "$1") || return 1
1188     _target=${2:-$_bin}
1189     [[ -e $initdir/$_target ]] && return 0
1190     [[ -L $_bin ]] && inst_symlink $_bin $_target && return 0
1191     local _file _line
1192     local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
1193     # I love bash!
1194     LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
1195         [[ $_line = 'not a dynamic executable' ]] && break
1196
1197         if [[ $_line =~ $_so_regex ]]; then
1198             _file=${BASH_REMATCH[1]}
1199             [[ -e ${initdir}/$_file ]] && continue
1200             inst_library "$_file"
1201             continue
1202         fi
1203
1204         if [[ $_line =~ not\ found ]]; then
1205             dfatal "Missing a shared library required by $_bin."
1206             dfatal "Run \"ldd $_bin\" to find out what it is."
1207             dfatal "$_line"
1208             dfatal "dracut cannot create an initrd."
1209             exit 1
1210         fi
1211     done
1212     inst_simple "$_bin" "$_target"
1213 }
1214
1215 # same as above, except for shell scripts.
1216 # If your shell script does not start with shebang, it is not a shell script.
1217 inst_script() {
1218     local _bin
1219     _bin=$(find_binary "$1") || return 1
1220     shift
1221     local _line _shebang_regex
1222     read -r -n 80 _line <"$_bin"
1223     # If debug is set, clean unprintable chars to prevent messing up the term
1224     [[ $debug ]] && _line=$(echo -n "$_line" | tr -c -d '[:print:][:space:]')
1225     _shebang_regex='(#! *)(/[^ ]+).*'
1226     [[ $_line =~ $_shebang_regex ]] || return 1
1227     inst "${BASH_REMATCH[2]}" && inst_simple "$_bin" "$@"
1228 }
1229
1230 # same as above, but specialized for symlinks
1231 inst_symlink() {
1232     local _src=$1 _target=${2:-$1} _realsrc
1233     strstr "$1" "/" || return 1
1234     [[ -L $1 ]] || return 1
1235     [[ -L $initdir/$_target ]] && return 0
1236     _realsrc=$(readlink -f "$_src")
1237     if ! [[ -e $initdir/$_realsrc ]]; then
1238         if [[ -d $_realsrc ]]; then
1239             inst_dir "$_realsrc"
1240         else
1241             inst "$_realsrc"
1242         fi
1243     fi
1244     [[ ! -e $initdir/${_target%/*} ]] && inst_dir "${_target%/*}"
1245     [[ -d ${_target%/*} ]] && _target=$(readlink -f ${_target%/*})/${_target##*/}
1246     ln -sfn $(convert_abs_rel "${_target}" "${_realsrc}") "$initdir/$_target"
1247 }
1248
1249 # attempt to install any programs specified in a udev rule
1250 inst_rule_programs() {
1251     local _prog _bin
1252
1253     if grep -qE 'PROGRAM==?"[^ "]+' "$1"; then
1254         for _prog in $(grep -E 'PROGRAM==?"[^ "]+' "$1" | sed -r 's/.*PROGRAM==?"([^ "]+).*/\1/'); do
1255             if [ -x /lib/udev/$_prog ]; then
1256                 _bin=/lib/udev/$_prog
1257             else
1258                 _bin=$(find_binary "$_prog") || {
1259                     dinfo "Skipping program $_prog using in udev rule $(basename $1) as it cannot be found"
1260                     continue;
1261                 }
1262             fi
1263
1264             #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
1265             dracut_install "$_bin"
1266         done
1267     fi
1268 }
1269
1270 # udev rules always get installed in the same place, so
1271 # create a function to install them to make life simpler.
1272 inst_rules() {
1273     local _target=/etc/udev/rules.d _rule _found
1274
1275     inst_dir "/lib/udev/rules.d"
1276     inst_dir "$_target"
1277     for _rule in "$@"; do
1278         if [ "${rule#/}" = "$rule" ]; then
1279             for r in /lib/udev/rules.d /etc/udev/rules.d; do
1280                 if [[ -f $r/$_rule ]]; then
1281                     _found="$r/$_rule"
1282                     inst_simple "$_found"
1283                     inst_rule_programs "$_found"
1284                 fi
1285             done
1286         fi
1287         for r in '' ./ $dracutbasedir/rules.d/; do
1288             if [[ -f ${r}$_rule ]]; then
1289                 _found="${r}$_rule"
1290                 inst_simple "$_found" "$_target/${_found##*/}"
1291                 inst_rule_programs "$_found"
1292             fi
1293         done
1294         [[ $_found ]] || dinfo "Skipping udev rule: $_rule"
1295         _found=
1296     done
1297 }
1298
1299 # general purpose installation function
1300 # Same args as above.
1301 inst() {
1302     local _x
1303
1304     case $# in
1305         1) ;;
1306         2) [[ ! $initdir && -d $2 ]] && export initdir=$2
1307             [[ $initdir = $2 ]] && set $1;;
1308         3) [[ -z $initdir ]] && export initdir=$2
1309             set $1 $3;;
1310         *) dfatal "inst only takes 1 or 2 or 3 arguments"
1311             exit 1;;
1312     esac
1313     for _x in inst_symlink inst_script inst_binary inst_simple; do
1314         $_x "$@" && return 0
1315     done
1316     return 1
1317 }
1318
1319 # install any of listed files
1320 #
1321 # If first argument is '-d' and second some destination path, first accessible
1322 # source is installed into this path, otherwise it will installed in the same
1323 # path as source.  If none of listed files was installed, function return 1.
1324 # On first successful installation it returns with 0 status.
1325 #
1326 # Example:
1327 #
1328 # inst_any -d /bin/foo /bin/bar /bin/baz
1329 #
1330 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
1331 # initramfs.
1332 inst_any() {
1333     local to f
1334
1335     [[ $1 = '-d' ]] && to="$2" && shift 2
1336
1337     for f in "$@"; do
1338         if [[ -e $f ]]; then
1339             [[ $to ]] && inst "$f" "$to" && return 0
1340             inst "$f" && return 0
1341         fi
1342     done
1343
1344     return 1
1345 }
1346
1347 # dracut_install [-o ] <file> [<file> ... ]
1348 # Install <file> to the initramfs image
1349 # -o optionally install the <file> and don't fail, if it is not there
1350 dracut_install() {
1351     local _optional=no
1352     if [[ $1 = '-o' ]]; then
1353         _optional=yes
1354         shift
1355     fi
1356     while (($# > 0)); do
1357         if ! inst "$1" ; then
1358             if [[ $_optional = yes ]]; then
1359                 dinfo "Skipping program $1 as it cannot be found and is" \
1360                     "flagged to be optional"
1361             else
1362                 dfatal "Failed to install $1"
1363                 exit 1
1364             fi
1365         fi
1366         shift
1367     done
1368 }
1369
1370 # Install a single kernel module along with any firmware it may require.
1371 # $1 = full path to kernel module to install
1372 install_kmod_with_fw() {
1373     # no need to go further if the module is already installed
1374
1375     [[ -e "${initdir}/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" ]] \
1376         && return 0
1377
1378     [[ -e "$initdir/.kernelmodseen/${1##*/}" ]] && return 0
1379
1380     if [[ $omit_drivers ]]; then
1381         local _kmod=${1##*/}
1382         _kmod=${_kmod%.ko}
1383         _kmod=${_kmod/-/_}
1384         if [[ "$_kmod" =~ $omit_drivers ]]; then
1385             dinfo "Omitting driver $_kmod"
1386             return 1
1387         fi
1388         if [[ "${1##*/lib/modules/$KERNEL_VER/}" =~ $omit_drivers ]]; then
1389             dinfo "Omitting driver $_kmod"
1390             return 1
1391         fi
1392     fi
1393
1394     [ -d "$initdir/.kernelmodseen" ] && \
1395         > "$initdir/.kernelmodseen/${1##*/}"
1396
1397     inst_simple "$1" "/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" \
1398         || return $?
1399
1400     local _modname=${1##*/} _fwdir _found _fw
1401     _modname=${_modname%.ko*}
1402     for _fw in $(modinfo -k $KERNEL_VER -F firmware $1 2>/dev/null); do
1403         _found=''
1404         for _fwdir in $fw_dir; do
1405             if [[ -d $_fwdir && -f $_fwdir/$_fw ]]; then
1406                 inst_simple "$_fwdir/$_fw" "/lib/firmware/$_fw"
1407                 _found=yes
1408             fi
1409         done
1410         if [[ $_found != yes ]]; then
1411             if ! grep -qe "\<${_modname//-/_}\>" /proc/modules; then
1412                 dinfo "Possible missing firmware \"${_fw}\" for kernel module" \
1413                     "\"${_modname}.ko\""
1414             else
1415                 dwarn "Possible missing firmware \"${_fw}\" for kernel module" \
1416                     "\"${_modname}.ko\""
1417             fi
1418         fi
1419     done
1420     return 0
1421 }
1422
1423 # Do something with all the dependencies of a kernel module.
1424 # Note that kernel modules depend on themselves using the technique we use
1425 # $1 = function to call for each dependency we find
1426 #      It will be passed the full path to the found kernel module
1427 # $2 = module to get dependencies for
1428 # rest of args = arguments to modprobe
1429 # _fderr specifies FD passed from surrounding scope
1430 for_each_kmod_dep() {
1431     local _func=$1 _kmod=$2 _cmd _modpath _options _found=0
1432     shift 2
1433     modprobe "$@" --ignore-install --show-depends $_kmod 2>&${_fderr} | (
1434         while read _cmd _modpath _options; do
1435             [[ $_cmd = insmod ]] || continue
1436             $_func ${_modpath} || exit $?
1437             _found=1
1438         done
1439         [[ $_found -eq 0 ]] && exit 1
1440         exit 0
1441     )
1442 }
1443
1444 # filter kernel modules to install certain modules that meet specific
1445 # requirements.
1446 # $1 = search only in subdirectory of /kernel/$1
1447 # $2 = function to call with module name to filter.
1448 #      This function will be passed the full path to the module to test.
1449 # The behavior of this function can vary depending on whether $hostonly is set.
1450 # If it is, we will only look at modules that are already in memory.
1451 # If it is not, we will look at all kernel modules
1452 # This function returns the full filenames of modules that match $1
1453 filter_kernel_modules_by_path () (
1454     local _modname _filtercmd
1455     if ! [[ $hostonly ]]; then
1456         _filtercmd='find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra"'
1457         _filtercmd+=' "$KERNEL_MODS/weak-updates" -name "*.ko" -o -name "*.ko.gz"'
1458         _filtercmd+=' -o -name "*.ko.xz"'
1459         _filtercmd+=' 2>/dev/null'
1460     else
1461         _filtercmd='cut -d " " -f 1 </proc/modules|xargs modinfo -F filename '
1462         _filtercmd+='-k $KERNEL_VER 2>/dev/null'
1463     fi
1464     for _modname in $(eval $_filtercmd); do
1465         case $_modname in
1466             *.ko) "$2" "$_modname" && echo "$_modname";;
1467             *.ko.gz) gzip -dc "$_modname" > $initdir/$$.ko
1468                 $2 $initdir/$$.ko && echo "$_modname"
1469                 rm -f $initdir/$$.ko
1470                 ;;
1471             *.ko.xz) xz -dc "$_modname" > $initdir/$$.ko
1472                 $2 $initdir/$$.ko && echo "$_modname"
1473                 rm -f $initdir/$$.ko
1474                 ;;
1475         esac
1476     done
1477 )
1478 find_kernel_modules_by_path () (
1479     if ! [[ $hostonly ]]; then
1480         find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra" "$KERNEL_MODS/weak-updates" \
1481           -name "*.ko" -o -name "*.ko.gz" -o -name "*.ko.xz" 2>/dev/null
1482     else
1483         cut -d " " -f 1 </proc/modules \
1484         | xargs modinfo -F filename -k $KERNEL_VER 2>/dev/null
1485     fi
1486 )
1487
1488 filter_kernel_modules () {
1489     filter_kernel_modules_by_path  drivers  "$1"
1490 }
1491
1492 find_kernel_modules () {
1493     find_kernel_modules_by_path  drivers
1494 }
1495
1496 # instmods [-c] <kernel module> [<kernel module> ... ]
1497 # instmods [-c] <kernel subsystem>
1498 # install kernel modules along with all their dependencies.
1499 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
1500 instmods() {
1501     [[ $no_kernel = yes ]] && return
1502     # called [sub]functions inherit _fderr
1503     local _fderr=9
1504     local _check=no
1505     if [[ $1 = '-c' ]]; then
1506         _check=yes
1507         shift
1508     fi
1509
1510     function inst1mod() {
1511         local _ret=0 _mod="$1"
1512         case $_mod in
1513             =*)
1514                 if [ -f $KERNEL_MODS/modules.${_mod#=} ]; then
1515                     ( [[ "$_mpargs" ]] && echo $_mpargs
1516                       cat "${KERNEL_MODS}/modules.${_mod#=}" ) \
1517                     | instmods
1518                 else
1519                     ( [[ "$_mpargs" ]] && echo $_mpargs
1520                       find "$KERNEL_MODS" -path "*/${_mod#=}/*" -printf '%f\n' ) \
1521                     | instmods
1522                 fi
1523                 ;;
1524             --*) _mpargs+=" $_mod" ;;
1525             i2o_scsi) return ;; # Do not load this diagnostic-only module
1526             *)
1527                 _mod=${_mod##*/}
1528                 # if we are already installed, skip this module and go on
1529                 # to the next one.
1530                 [[ -f "$initdir/.kernelmodseen/${_mod%.ko}.ko" ]] && return
1531
1532                 if [[ $omit_drivers ]] && [[ "$1" =~ $omit_drivers ]]; then
1533                     dinfo "Omitting driver ${_mod##$KERNEL_MODS}"
1534                     return
1535                 fi
1536                 # If we are building a host-specific initramfs and this
1537                 # module is not already loaded, move on to the next one.
1538                 [[ $hostonly ]] && ! grep -qe "\<${_mod//-/_}\>" /proc/modules \
1539                     && ! echo $add_drivers | grep -qe "\<${_mod}\>" \
1540                     && return
1541
1542                 # We use '-d' option in modprobe only if modules prefix path
1543                 # differs from default '/'.  This allows us to use Dracut with
1544                 # old version of modprobe which doesn't have '-d' option.
1545                 local _moddirname=${KERNEL_MODS%%/lib/modules/*}
1546                 [[ -n ${_moddirname} ]] && _moddirname="-d ${_moddirname}/"
1547
1548                 # ok, load the module, all its dependencies, and any firmware
1549                 # it may require
1550                 for_each_kmod_dep install_kmod_with_fw $_mod \
1551                     --set-version $KERNEL_VER ${_moddirname} $_mpargs
1552                 ((_ret+=$?))
1553                 ;;
1554         esac
1555         return $_ret
1556     }
1557
1558     function instmods_1() {
1559         local _mod _mpargs
1560         if (($# == 0)); then  # filenames from stdin
1561             while read _mod; do
1562                 inst1mod "${_mod%.ko*}" || {
1563                     if [ "$_check" = "yes" ]; then
1564                         dfatal "Failed to install $_mod"
1565                         return 1
1566                     fi
1567                 }
1568             done
1569         fi
1570         while (($# > 0)); do  # filenames as arguments
1571             inst1mod ${1%.ko*} || {
1572                 if [ "$_check" = "yes" ]; then
1573                     dfatal "Failed to install $1"
1574                     return 1
1575                 fi
1576             }
1577             shift
1578         done
1579         return 0
1580     }
1581
1582     local _ret _filter_not_found='FATAL: Module .* not found.'
1583     set -o pipefail
1584     # Capture all stderr from modprobe to _fderr. We could use {var}>...
1585     # redirections, but that would make dracut require bash4 at least.
1586     eval "( instmods_1 \"\$@\" ) ${_fderr}>&1" \
1587     | while read line; do [[ "$line" =~ $_filter_not_found ]] && echo $line || echo $line >&2 ;done | derror
1588     _ret=$?
1589     set +o pipefail
1590     return $_ret
1591 }
1592
1593 # inst_libdir_file [-n <pattern>] <file> [<file>...]
1594 # Install a <file> located on a lib directory to the initramfs image
1595 # -n <pattern> install non-matching files
1596 inst_libdir_file() {
1597     if [[ "$1" == "-n" ]]; then
1598         local _pattern=$1
1599         shift 2
1600         for _dir in $libdirs; do
1601             for _i in "$@"; do
1602                 for _f in "$_dir"/$_i; do
1603                     [[ "$_i" =~ $_pattern ]] || continue
1604                     [[ -e "$_i" ]] && dracut_install "$_i"
1605                 done
1606             done
1607         done
1608     else
1609         for _dir in $libdirs; do
1610             for _i in "$@"; do
1611                 for _f in "$_dir"/$_i; do
1612                     [[ -e "$_f" ]] && dracut_install "$_f"
1613                 done
1614             done
1615         done
1616     fi
1617 }
1618
1619 setup_suse() {
1620     ln -fs ../usr/bin/systemctl $initdir/bin/
1621     ln -fs ../usr/lib/systemd $initdir/lib/
1622     inst_simple "/usr/lib/systemd/system/haveged.service"
1623 }
1624
1625 # can be overridden in specific test
1626 test_cleanup() {
1627     umount $TESTDIR/root 2>/dev/null || true
1628     [[ $LOOPDEV ]] && losetup -d $LOOPDEV || true
1629     return 0
1630 }
1631
1632 test_run() {
1633     if [ -z "$TEST_NO_QEMU" ]; then
1634         if run_qemu; then
1635             check_result_qemu || return 1
1636         else
1637             dwarn "can't run QEMU, skipping"
1638         fi
1639     fi
1640     if [ -z "$TEST_NO_NSPAWN" ]; then
1641         if run_nspawn "nspawn-root"; then
1642             check_result_nspawn "nspawn-root" || return 1
1643         else
1644             dwarn "can't run systemd-nspawn, skipping"
1645         fi
1646
1647         if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
1648             if NSPAWN_ARGUMENTS="-U --private-network $NSPAWN_ARGUMENTS" run_nspawn "unprivileged-nspawn-root"; then
1649                 check_result_nspawn "unprivileged-nspawn-root" || return 1
1650             else
1651                 dwarn "can't run systemd-nspawn, skipping"
1652             fi
1653         fi
1654     fi
1655     return 0
1656 }
1657
1658 do_test() {
1659     if [[ $UID != "0" ]]; then
1660         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
1661         exit 0
1662     fi
1663
1664     # Detect lib paths
1665     [[ $libdir ]] || for libdir in /lib64 /lib; do
1666         [[ -d $libdir ]] && libdirs+=" $libdir" && break
1667     done
1668
1669     [[ $usrlibdir ]] || for usrlibdir in /usr/lib64 /usr/lib; do
1670         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
1671     done
1672
1673     mkdir -p "$STATEDIR"
1674
1675     import_testdir
1676     import_initdir
1677
1678     while (($# > 0)); do
1679         case $1 in
1680             --run)
1681                 echo "TEST RUN: $TEST_DESCRIPTION"
1682                 test_run
1683                 ret=$?
1684                 if (( $ret == 0 )); then
1685                     echo "TEST RUN: $TEST_DESCRIPTION [OK]"
1686                 else
1687                     echo "TEST RUN: $TEST_DESCRIPTION [FAILED]"
1688                 fi
1689                 exit $ret;;
1690             --setup)
1691                 echo "TEST SETUP: $TEST_DESCRIPTION"
1692                 test_setup
1693                 ;;
1694             --clean)
1695                 echo "TEST CLEANUP: $TEST_DESCRIPTION"
1696                 test_cleanup
1697                 rm -fr "$TESTDIR"
1698                 rm -f "$STATEFILE"
1699                 ;;
1700             --all)
1701                 ret=0
1702                 echo -n "TEST: $TEST_DESCRIPTION ";
1703                 (
1704                     test_setup && test_run
1705                     ret=$?
1706                     test_cleanup
1707                     rm -fr "$TESTDIR"
1708                     rm -f "$STATEFILE"
1709                     exit $ret
1710                 ) </dev/null >"$TESTLOG" 2>&1 || ret=$?
1711                 if [ $ret -eq 0 ]; then
1712                     rm "$TESTLOG"
1713                     echo "[OK]"
1714                 else
1715                     echo "[FAILED]"
1716                     echo "see $TESTLOG"
1717                 fi
1718                 exit $ret;;
1719             *) break ;;
1720         esac
1721         shift
1722     done
1723 }