test/test-functions

   1 #!/bin/bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # ex: ts=8 sw=4 sts=4 et filetype=sh
   4 PATH=/sbin:/bin:/usr/sbin:/usr/bin
   5 export PATH
   6
   7 LOOKS_LIKE_DEBIAN=$(source /etc/os-release && [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && echo yes || true)
   8 LOOKS_LIKE_ARCH=$(source /etc/os-release && [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && echo yes || true)
   9 LOOKS_LIKE_SUSE=$(source /etc/os-release && [[ " $ID_LIKE " = *" suse "* ]] && echo yes || true)
  10 KERNEL_VER=${KERNEL_VER-$(uname -r)}
  11 KERNEL_MODS="/lib/modules/$KERNEL_VER/"
  12 QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
  13 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
  14 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  15 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  16 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  17 EFI_MOUNT="$(bootctl -p 2>/dev/null || echo /boot)"
  18 QEMU_MEM="${QEMU_MEM:-512M}"
  19
  20 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  21     echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
  22     ROOTLIBDIR=/usr/lib/systemd
  23 fi
  24
  25 PATH_TO_INIT=$ROOTLIBDIR/systemd
  26
  27 BASICTOOLS="test sh bash setsid loadkeys setfont login sulogin gzip sleep echo head tail cat mount umount cryptsetup date dmsetup modprobe sed cmp tee rm true false chmod chown ln xargs"
  28 DEBUGTOOLS="df free ls stty ps ln ip route dmesg dhclient mkdir cp ping dhclient strace less grep id tty touch du sort hostname find vi mv"
  29
  30 STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))"
  31 STATEFILE="$STATEDIR/.testdir"
  32 TESTLOG="$STATEDIR/test.log"
  33
  34 is_built_with_asan() {
  35     if ! type -P objdump >/dev/null; then
  36         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
  37         return 1
  38     fi
  39
  40     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
  41     local _asan_calls=$(objdump -dC $BUILD_DIR/systemd-journald | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
  42     if (( $_asan_calls < 1000 )); then
  43         return 1
  44     else
  45         return 0
  46     fi
  47 }
  48
  49 IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)
  50
  51 if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
  52     STRIP_BINARIES=no
  53     SKIP_INITRD="${SKIP_INITRD:-yes}"
  54     PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
  55     QEMU_MEM="1536M"
  56     QEMU_SMP=4
  57 fi
  58
  59 function find_qemu_bin() {
  60     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
  61     # Either way, only use this version if we aren't running in KVM, because
  62     # nested KVM is flaky still.
  63     if [[ $(systemd-detect-virt -v) != kvm && -z $TEST_NO_KVM ]] ; then
  64         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a kvm qemu-kvm 2>/dev/null | grep '^/' -m1)
  65     fi
  66
  67     [ "$ARCH" ] || ARCH=$(uname -m)
  68     case $ARCH in
  69     x86_64)
  70         # QEMU's own build system calls it qemu-system-x86_64
  71         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-x86_64 2>/dev/null | grep '^/' -m1)
  72         ;;
  73     i*86)
  74         # new i386 version of QEMU
  75         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-i386 2>/dev/null | grep '^/' -m1)
  76
  77         # i386 version of QEMU
  78         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu 2>/dev/null | grep '^/' -m1)
  79         ;;
  80     ppc64*)
  81         [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-$ARCH 2>/dev/null | grep '^/' -m1)
  82         ;;
  83     esac
  84
  85     if [ ! -e "$QEMU_BIN" ]; then
  86         echo "Could not find a suitable QEMU binary" >&2
  87         return 1
  88     fi
  89 }
  90
  91 # Return 0 if QEMU did run (then you must check the result state/logs for actual
  92 # success), or 1 if QEMU is not available.
  93 run_qemu() {
  94     if [ -f /etc/machine-id ]; then
  95         read MACHINE_ID < /etc/machine-id
  96         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
  97             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
  98         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
  99             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 100     fi
 101
 102     CONSOLE=ttyS0
 103
 104     if [[ ! "$KERNEL_BIN" ]]; then
 105         if [[ "$LOOKS_LIKE_ARCH" ]]; then
 106             KERNEL_BIN=/boot/vmlinuz-linux
 107         else
 108             [ "$ARCH" ] || ARCH=$(uname -m)
 109             case $ARCH in
 110                 ppc64*)
 111                 KERNEL_BIN=/boot/vmlinux-$KERNEL_VER
 112                 CONSOLE=hvc0
 113                 ;;
 114                 *)
 115                 KERNEL_BIN=/boot/vmlinuz-$KERNEL_VER
 116                 ;;
 117             esac
 118         fi
 119     fi
 120
 121     default_fedora_initrd=/boot/initramfs-${KERNEL_VER}.img
 122     default_debian_initrd=/boot/initrd.img-${KERNEL_VER}
 123     default_arch_initrd=/boot/initramfs-linux-fallback.img
 124     default_suse_initrd=/boot/initrd-${KERNEL_VER}
 125     if [[ ! "$INITRD" ]]; then
 126         if [[ -e "$default_fedora_initrd" ]]; then
 127             INITRD="$default_fedora_initrd"
 128         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 129             INITRD="$default_debian_initrd"
 130         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 131             INITRD="$default_arch_initrd"
 132         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 133             INITRD="$default_suse_initrd"
 134         fi
 135     fi
 136
 137     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 138     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 139     # is not installed or there's some other error when calling it, fall back
 140     # to the original value (QEMU_SMP=1).
 141     if ! [ "$QEMU_SMP" ]; then
 142         if ! QEMU_SMP=$(nproc); then
 143             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 144             QEMU_SMP=1
 145         fi
 146     fi
 147
 148     find_qemu_bin || return 1
 149
 150     local _cgroup_args
 151     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 152         _cgroup_args="systemd.unified_cgroup_hierarchy=yes"
 153     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 154         _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=yes"
 155     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 156         _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=no"
 157     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 158         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 159         exit 1
 160     fi
 161
 162     if [[ "$LOOKS_LIKE_SUSE" ]]; then
 163         PARAMS+="rd.hostonly=0"
 164     elif [[ "$LOOKS_LIKE_ARCH" ]]; then
 165         PARAMS+="rw"
 166     else
 167         PARAMS+="ro"
 168     fi
 169
 170     KERNEL_APPEND="$PARAMS \
 171 root=/dev/sda1 \
 172 raid=noautodetect \
 173 loglevel=2 \
 174 init=$PATH_TO_INIT \
 175 console=$CONSOLE \
 176 selinux=0 \
 177 printk.devkmsg=on \
 178 $_cgroup_args \
 179 $KERNEL_APPEND \
 180 "
 181
 182     QEMU_OPTIONS="-smp $QEMU_SMP \
 183 -net none \
 184 -m $QEMU_MEM \
 185 -nographic \
 186 -kernel $KERNEL_BIN \
 187 -drive format=raw,cache=unsafe,file=${TESTDIR}/rootdisk.img \
 188 "
 189
 190     if [[ "$INITRD" && "$SKIP_INITRD" != "yes" ]]; then
 191         QEMU_OPTIONS="$QEMU_OPTIONS -initrd $INITRD"
 192     fi
 193
 194     # Let's use KVM if it is available, but let's avoid using nested KVM as that is still flaky
 195     if [[ -c /dev/kvm && $(systemd-detect-virt -v) != kvm && -z $TEST_NO_KVM ]] ; then
 196         QEMU_OPTIONS="$QEMU_OPTIONS -machine accel=kvm -enable-kvm -cpu host"
 197     fi
 198
 199     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 200         QEMU_BIN="timeout --foreground $QEMU_TIMEOUT $QEMU_BIN"
 201     fi
 202     (set -x; $QEMU_BIN $QEMU_OPTIONS -append "$KERNEL_APPEND")
 203     rc=$?
 204     if [ "$rc" = 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 205         derror "test timed out after $QEMU_TIMEOUT s"
 206         TIMED_OUT=1
 207     else
 208         [ "$rc" != 0 ] && derror "QEMU failed with exit code $rc"
 209     fi
 210     return 0
 211 }
 212
 213 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 214 # success), or 1 if nspawn is not available.
 215 run_nspawn() {
 216     [[ -d /run/systemd/system ]] || return 1
 217
 218     local _nspawn_cmd="$BUILD_DIR/systemd-nspawn $NSPAWN_ARGUMENTS --register=no --kill-signal=SIGKILL --directory=$TESTDIR/$1 $PATH_TO_INIT $KERNEL_APPEND"
 219     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 220         _nspawn_cmd="timeout --foreground $NSPAWN_TIMEOUT $_nspawn_cmd"
 221     fi
 222
 223     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 224         dwarn "nspawn doesn't support UNIFIED_CGROUP_HIERARCHY=hybrid, skipping"
 225         exit
 226     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 227         _nspawn_cmd="env UNIFIED_CGROUP_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY $_nspawn_cmd"
 228     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 229         _nspawn_cmd="env --unset=UNIFIED_CGROUP_HIERARCHY $_nspawn_cmd"
 230     else
 231         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 232         exit 1
 233     fi
 234
 235     (set -x; $_nspawn_cmd)
 236     rc=$?
 237     if [ "$rc" = 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 238         derror "test timed out after $NSPAWN_TIMEOUT s"
 239         TIMED_OUT=1
 240     else
 241         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 242     fi
 243     return 0
 244 }
 245
 246 setup_basic_environment() {
 247     # create the basic filesystem layout
 248     setup_basic_dirs
 249
 250     install_systemd
 251     install_missing_libraries
 252     install_config_files
 253     create_rc_local
 254     install_basic_tools
 255     install_libnss
 256     install_pam
 257     install_dbus
 258     install_fonts
 259     install_keymaps
 260     install_terminfo
 261     install_execs
 262     install_fsck
 263     install_plymouth
 264     install_debug_tools
 265     install_ld_so_conf
 266     setup_selinux
 267     strip_binaries
 268     install_depmod_files
 269     generate_module_dependencies
 270     if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
 271          create_asan_wrapper
 272     fi
 273 }
 274
 275 setup_selinux() {
 276     # don't forget KERNEL_APPEND='... selinux=1 ...'
 277     if [[ "$SETUP_SELINUX" != "yes" ]]; then
 278         ddebug "Don't setup SELinux"
 279         return 0
 280     fi
 281     ddebug "Setup SELinux"
 282     local _conf_dir=/etc/selinux
 283     local _fixfiles_tools="bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles"
 284
 285     rm -rf $initdir/$_conf_dir
 286     if ! cp -ar $_conf_dir $initdir/$_conf_dir; then
 287         dfatal "Failed to copy $_conf_dir"
 288         exit 1
 289     fi
 290
 291     cat <<EOF >$initdir/etc/systemd/system/autorelabel.service
 292 [Unit]
 293 Description=Relabel all filesystems
 294 DefaultDependencies=no
 295 Requires=local-fs.target
 296 Conflicts=shutdown.target
 297 After=local-fs.target
 298 Before=sysinit.target shutdown.target
 299 ConditionSecurity=selinux
 300 ConditionPathExists=|/.autorelabel
 301
 302 [Service]
 303 ExecStart=/bin/sh -x -c 'echo 0 >/sys/fs/selinux/enforce && fixfiles -f -F relabel && rm /.autorelabel && systemctl --force reboot'
 304 Type=oneshot
 305 TimeoutSec=0
 306 RemainAfterExit=yes
 307 EOF
 308
 309     touch $initdir/.autorelabel
 310     mkdir -p $initdir/etc/systemd/system/basic.target.wants
 311     ln -fs autorelabel.service $initdir/etc/systemd/system/basic.target.wants/autorelabel.service
 312
 313     dracut_install $_fixfiles_tools
 314     dracut_install fixfiles
 315     dracut_install sestatus
 316 }
 317
 318 install_valgrind() {
 319     if ! type -p valgrind; then
 320         dfatal "Failed to install valgrind"
 321         exit 1
 322     fi
 323
 324     local _valgrind_bins=$(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')
 325     dracut_install $_valgrind_bins
 326
 327     local _valgrind_libs=$(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')
 328     dracut_install $_valgrind_libs
 329
 330     local _valgrind_dbg_and_supp=$(
 331         strace -e open valgrind /bin/true 2>&1 >/dev/null |
 332         perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
 333     )
 334     dracut_install $_valgrind_dbg_and_supp
 335 }
 336
 337 create_valgrind_wrapper() {
 338     local _valgrind_wrapper=$initdir/$ROOTLIBDIR/systemd-under-valgrind
 339     ddebug "Create $_valgrind_wrapper"
 340     cat >$_valgrind_wrapper <<EOF
 341 #!/bin/bash
 342
 343 mount -t proc proc /proc
 344 exec valgrind --leak-check=full --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 345 EOF
 346     chmod 0755 $_valgrind_wrapper
 347 }
 348
 349 create_asan_wrapper() {
 350     local _asan_wrapper=$initdir/$ROOTLIBDIR/systemd-under-asan
 351     ddebug "Create $_asan_wrapper"
 352     cat >$_asan_wrapper <<EOF
 353 #!/bin/bash
 354
 355 set -x
 356
 357 DEFAULT_ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
 358 DEFAULT_UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
 359 DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS"
 360
 361 mount -t proc proc /proc
 362 mount -t sysfs sysfs /sys
 363 mount -o remount,rw /
 364
 365 PATH_TO_ASAN=\$(find / -name '*libasan*' | sed 1q)
 366 if [[ "\$PATH_TO_ASAN" ]]; then
 367   # A lot of services (most notably dbus) won't start without preloading libasan
 368   # See https://github.com/systemd/systemd/issues/5004
 369   DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=\$PATH_TO_ASAN"
 370 fi
 371 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
 372 echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 373
 374 # ASAN and syscall filters aren't compatible with each other.
 375 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
 376
 377 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
 378 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
 379 JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 380 mkdir -p "\$JOURNALD_CONF_DIR"
 381 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 382
 383 # 90s isn't enough for some services to finish when literally everything is run
 384 # under ASan+UBSan in containers, which, in turn, are run in VMs.
 385 # Let's limit which environments such services should be executed in.
 386 mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
 387 printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
 388
 389 # The 'mount' utility doesn't behave well under libasan, causing unexpected
 390 # fails during boot and subsequent test results check:
 391 # bash-5.0# mount -o remount,rw -v /
 392 # mount: /dev/sda1 mounted on /.
 393 # bash-5.0# echo \$?
 394 # 1
 395 # Let's workaround this by clearing the previously set LD_PRELOAD env variable,
 396 # so the libasan library is not loaded for this particular service
 397 REMOUNTFS_CONF_DIR=/etc/systemd/system/systemd-remount-fs.service.d
 398 mkdir -p "\$REMOUNTFS_CONF_DIR"
 399 printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$REMOUNTFS_CONF_DIR/env.conf"
 400
 401 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 402 exec  $ROOTLIBDIR/systemd "\$@"
 403 EOF
 404
 405     chmod 0755 $_asan_wrapper
 406 }
 407
 408 create_strace_wrapper() {
 409     local _strace_wrapper=$initdir/$ROOTLIBDIR/systemd-under-strace
 410     ddebug "Create $_strace_wrapper"
 411     cat >$_strace_wrapper <<EOF
 412 #!/bin/bash
 413
 414 exec strace -D -o /strace.out $ROOTLIBDIR/systemd "\$@"
 415 EOF
 416     chmod 0755 $_strace_wrapper
 417 }
 418
 419 install_fsck() {
 420     dracut_install /sbin/fsck*
 421     dracut_install -o /bin/fsck*
 422
 423     # fskc.reiserfs calls reiserfsck. so, install it
 424     dracut_install -o reiserfsck
 425 }
 426
 427 install_dmevent() {
 428     instmods dm_crypt =crypto
 429     type -P dmeventd >/dev/null && dracut_install dmeventd
 430     inst_libdir_file "libdevmapper-event.so*"
 431     if [[ "$LOOKS_LIKE_DEBIAN" ]]; then
 432         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
 433         # and since buster/bionic 95-dm-notify.rules
 434         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
 435         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
 436     else
 437         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
 438     fi
 439 }
 440
 441 install_systemd() {
 442     # install compiled files
 443     local _ninja_bin=$(type -P ninja || type -P ninja-build)
 444     if [[ -z "$_ninja_bin" ]]; then
 445         dfatal "ninja was not found"
 446         exit 1
 447     fi
 448     (set -x; DESTDIR=$initdir "$_ninja_bin" -C $BUILD_DIR install)
 449     # remove unneeded documentation
 450     rm -fr $initdir/usr/share/{man,doc}
 451     # we strip binaries since debug symbols increase binaries size a lot
 452     # and it could fill the available space
 453     strip_binaries
 454
 455    [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse
 456
 457     # enable debug logging in PID1
 458     echo LogLevel=debug >> $initdir/etc/systemd/system.conf
 459 }
 460
 461 get_ldpath() {
 462     local _bin="$1"
 463     objdump -p "$_bin" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :
 464 }
 465
 466 install_missing_libraries() {
 467     # install possible missing libraries
 468     for i in $initdir{,/usr}/{sbin,bin}/* $initdir{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
 469         LD_LIBRARY_PATH=$(get_ldpath $i) inst_libs $i
 470     done
 471 }
 472
 473 create_empty_image() {
 474     local _size=500
 475     if [[ "$STRIP_BINARIES" = "no" ]]; then
 476         _size=$((2*_size))
 477     fi
 478     rm -f "$TESTDIR/rootdisk.img"
 479     # Create the blank file to use as a root filesystem
 480     dd if=/dev/null of="$TESTDIR/rootdisk.img" bs=1M seek="$_size"
 481     LOOPDEV=$(losetup --show -P -f $TESTDIR/rootdisk.img)
 482     [ -b "$LOOPDEV" ] || return 1
 483     echo "LOOPDEV=$LOOPDEV" >> $STATEFILE
 484     sfdisk "$LOOPDEV" <<EOF
 485 ,$((_size-50))M
 486 ,
 487 EOF
 488
 489     udevadm settle
 490
 491     local _label="-L systemd"
 492     # mkfs.reiserfs doesn't know -L. so, use --label instead
 493     [[ "$FSTYPE" == "reiserfs" ]] && _label="--label systemd"
 494     if ! mkfs -t "${FSTYPE}" ${_label} "${LOOPDEV}p1" -q; then
 495         dfatal "Failed to mkfs -t ${FSTYPE}"
 496         exit 1
 497     fi
 498 }
 499
 500 check_asan_reports() {
 501     local ret=0
 502     local root="$1"
 503
 504     if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
 505         ls -l "$root"
 506         if [[ -e "$root/systemd.asan.log.1" ]]; then
 507             cat "$root/systemd.asan.log.1"
 508             ret=$(($ret+1))
 509         fi
 510
 511         journald_report=$(find "$root" -name "systemd-journald.asan.log*" -exec cat {} \;)
 512         if [[ ! -z "$journald_report" ]]; then
 513             printf "%s" "$journald_report"
 514             ret=$(($ret+1))
 515         fi
 516
 517         pids=$(
 518             "$BUILD_DIR/journalctl" -D "$root/var/log/journal" | perl -alne '
 519                  BEGIN {
 520                      %services_to_ignore = (
 521                          "dbus-daemon" => undef,
 522                      );
 523                  }
 524                  print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
 525         )
 526         if [[ ! -z "$pids" ]]; then
 527             ret=$(($ret+1))
 528             for pid in $pids; do
 529                 "$BUILD_DIR/journalctl" -D "$root/var/log/journal" _PID=$pid --no-pager
 530             done
 531         fi
 532     fi
 533
 534     return $ret
 535 }
 536
 537 check_result_nspawn() {
 538     local ret=1
 539     local journald_report=""
 540     local pids=""
 541     [[ -e $TESTDIR/$1/testok ]] && ret=0
 542     [[ -f $TESTDIR/$1/failed ]] && cp -a $TESTDIR/$1/failed $TESTDIR
 543     cp -a $TESTDIR/$1/var/log/journal $TESTDIR
 544     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
 545     ls -l $TESTDIR/journal/*/*.journal
 546     test -s $TESTDIR/failed && ret=$(($ret+1))
 547     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
 548     check_asan_reports "$TESTDIR/$1" || ret=$(($ret+1))
 549     return $ret
 550 }
 551
 552 # can be overridden in specific test
 553 check_result_qemu() {
 554     local ret=1
 555     mkdir -p $TESTDIR/root
 556     mount ${LOOPDEV}p1 $TESTDIR/root
 557     [[ -e $TESTDIR/root/testok ]] && ret=0
 558     [[ -f $TESTDIR/root/failed ]] && cp -a $TESTDIR/root/failed $TESTDIR
 559     cp -a $TESTDIR/root/var/log/journal $TESTDIR
 560     check_asan_reports "$TESTDIR/root" || ret=$(($ret+1))
 561     umount $TESTDIR/root
 562     [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
 563     ls -l $TESTDIR/journal/*/*.journal
 564     test -s $TESTDIR/failed && ret=$(($ret+1))
 565     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
 566     return $ret
 567 }
 568
 569 strip_binaries() {
 570     if [[ "$STRIP_BINARIES" = "no" ]]; then
 571         ddebug "Don't strip binaries"
 572         return 0
 573     fi
 574     ddebug "Strip binaries"
 575     find "$initdir" -executable -not -path '*/lib/modules/*.ko' -type f | \
 576         xargs strip --strip-unneeded |& \
 577         grep -v 'file format not recognized' | \
 578         ddebug
 579 }
 580
 581 create_rc_local() {
 582     mkdir -p $initdir/etc/rc.d
 583     cat >$initdir/etc/rc.d/rc.local <<EOF
 584 #!/bin/bash
 585 exit 0
 586 EOF
 587     chmod 0755 $initdir/etc/rc.d/rc.local
 588 }
 589
 590 install_execs() {
 591     ddebug "install any Execs from the service files"
 592     (
 593     export PKG_CONFIG_PATH=$BUILD_DIR/src/core/
 594     systemdsystemunitdir=$(pkg-config --variable=systemdsystemunitdir systemd)
 595     systemduserunitdir=$(pkg-config --variable=systemduserunitdir systemd)
 596     sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' $initdir/{$systemdsystemunitdir,$systemduserunitdir}/*.service \
 597          | sort -u | while read i; do
 598          # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
 599          # also, plymouth is pulled in by rescue.service, but even there the exit code
 600          # is ignored; as it's not present on some distros, don't fail if it doesn't exist
 601          inst $i || [ "${i%.local}" != "$i" ] || [ "${i%systemd-update-done}" != "$i" ] || [ "/bin/plymouth" == "$i" ]
 602      done
 603     )
 604 }
 605
 606 generate_module_dependencies() {
 607     if [[ -d $initdir/lib/modules/$KERNEL_VER ]] && \
 608         ! depmod -a -b "$initdir" $KERNEL_VER; then
 609             dfatal "\"depmod -a $KERNEL_VER\" failed."
 610             exit 1
 611     fi
 612 }
 613
 614 install_depmod_files() {
 615     inst /lib/modules/$KERNEL_VER/modules.order
 616     inst /lib/modules/$KERNEL_VER/modules.builtin
 617 }
 618
 619 install_plymouth() {
 620     # install plymouth, if found... else remove plymouth service files
 621     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
 622     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
 623     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
 624     #         dracut_install plymouth plymouthd
 625     # else
 626         rm -f $initdir/{usr/lib,etc}/systemd/system/plymouth* $initdir/{usr/lib,etc}/systemd/system/*/plymouth*
 627     # fi
 628 }
 629
 630 install_ld_so_conf() {
 631     cp -a /etc/ld.so.conf* $initdir/etc
 632     ldconfig -r "$initdir"
 633 }
 634
 635 install_config_files() {
 636     inst /etc/sysconfig/init || true
 637     inst /etc/passwd
 638     inst /etc/shadow
 639     inst /etc/login.defs
 640     inst /etc/group
 641     inst /etc/shells
 642     inst /etc/nsswitch.conf
 643     inst /etc/pam.conf || true
 644     inst /etc/securetty || true
 645     inst /etc/os-release
 646     inst /etc/localtime
 647     # we want an empty environment
 648     > $initdir/etc/environment
 649     > $initdir/etc/machine-id
 650     # set the hostname
 651     echo systemd-testsuite > $initdir/etc/hostname
 652     # fstab
 653     if [[ "$LOOKS_LIKE_SUSE" ]]; then
 654        ROOTMOUNT="/dev/sda1           /       ${FSTYPE}    rw 0 1"
 655     else
 656        ROOTMOUNT="LABEL=systemd           /       ${FSTYPE}    rw 0 1"
 657     fi
 658
 659     cat >$initdir/etc/fstab <<EOF
 660 $ROOTMOUNT
 661 EOF
 662 }
 663
 664 install_basic_tools() {
 665     [[ $BASICTOOLS ]] && dracut_install $BASICTOOLS
 666     dracut_install -o sushell
 667     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
 668     dracut_install -o ldconfig.real
 669 }
 670
 671 install_debug_tools() {
 672     [[ $DEBUGTOOLS ]] && dracut_install $DEBUGTOOLS
 673
 674     if [[ $INTERACTIVE_DEBUG ]]; then
 675         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
 676         local _getty_override="$initdir/etc/systemd/system/serial-getty@.service.d"
 677         mkdir -p "$_getty_override"
 678         echo -e "[Service]\nEnvironment=TERM=linux" > "$_getty_override/default-TERM.conf"
 679
 680         cat > "$initdir/etc/motd" << EOF
 681 To adjust the terminal size use:
 682     export COLUMNS=xx
 683     export LINES=yy
 684 or
 685     stty cols xx rows yy
 686 EOF
 687     fi
 688 }
 689
 690 install_libnss() {
 691     # install libnss_files for login
 692     NSS_LIBS=$(LD_DEBUG=files getent passwd 2>&1 >/dev/null |sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
 693     dracut_install $NSS_LIBS
 694 }
 695
 696 install_dbus() {
 697     inst $ROOTLIBDIR/system/dbus.socket
 698
 699     # Fedora rawhide replaced dbus.service with dbus-daemon.service
 700     if [ -f $ROOTLIBDIR/system/dbus-daemon.service ]; then
 701         inst $ROOTLIBDIR/system/dbus-daemon.service
 702         # Alias symlink
 703         inst_symlink /etc/systemd/system/dbus.service
 704     else
 705         inst $ROOTLIBDIR/system/dbus.service
 706     fi
 707     # Newer Fedora versions use dbus-broker by default. Let's install it is available.
 708     [ -f /usr/bin/dbus-broker ] && inst /usr/bin/dbus-broker
 709     [ -f /usr/bin/dbus-broker-launch ] && inst /usr/bin/dbus-broker-launch
 710
 711     find \
 712         /etc/dbus-1 /usr/share/dbus-1 -xtype f \
 713         | while read file; do
 714         inst $file
 715     done
 716 }
 717
 718 install_pam() {
 719     (
 720     if [[ "$LOOKS_LIKE_DEBIAN" ]] && type -p dpkg-architecture &>/dev/null; then
 721         find "/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security" -xtype f
 722     else
 723         find /lib*/security -xtype f
 724     fi
 725     find /etc/pam.d /etc/security -xtype f
 726     ) | while read file; do
 727         inst $file
 728     done
 729
 730     # pam_unix depends on unix_chkpwd.
 731     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
 732     dracut_install -o unix_chkpwd
 733
 734     [[ "$LOOKS_LIKE_DEBIAN" ]] &&
 735         cp /etc/pam.d/systemd-user $initdir/etc/pam.d/
 736
 737     # set empty root password for easy debugging
 738     sed -i 's/^root:x:/root::/' $initdir/etc/passwd
 739 }
 740
 741 install_keymaps() {
 742     # The first three paths may be deprecated.
 743     # It seems now the last two paths are used by many distributions.
 744     for i in \
 745         /usr/lib/kbd/keymaps/include/* \
 746         /usr/lib/kbd/keymaps/i386/include/* \
 747         /usr/lib/kbd/keymaps/i386/qwerty/us.* \
 748         /usr/lib/kbd/keymaps/legacy/include/* \
 749         /usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
 750             [[ -f $i ]] || continue
 751             inst $i
 752     done
 753
 754     # When it takes any argument, then install more keymaps.
 755     if [[ -n $1 ]]; then
 756         for i in \
 757         /usr/lib/kbd/keymaps/i386/*/* \
 758         /usr/lib/kbd/keymaps/legacy/i386/*/*; do
 759             [[ -f $i ]] || continue
 760             inst $i
 761         done
 762     fi
 763 }
 764
 765 install_zoneinfo() {
 766     for i in /usr/share/zoneinfo/{,*/,*/*/}*; do
 767         [[ -f $i ]] || continue
 768         inst $i
 769     done
 770 }
 771
 772 install_fonts() {
 773     for i in \
 774         /usr/lib/kbd/consolefonts/eurlatgr* \
 775         /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
 776             [[ -f $i ]] || continue
 777             inst $i
 778     done
 779 }
 780
 781 install_terminfo() {
 782     for _terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
 783         [ -f ${_terminfodir}/l/linux ] && break
 784     done
 785     dracut_install -o ${_terminfodir}/l/linux
 786 }
 787
 788 setup_testsuite() {
 789     cp $TEST_BASE_DIR/testsuite.target $initdir/etc/systemd/system/
 790     cp $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/
 791
 792     mkdir -p $initdir/etc/systemd/system/testsuite.target.wants
 793     ln -fs $TEST_BASE_DIR/testsuite.service $initdir/etc/systemd/system/testsuite.target.wants/testsuite.service
 794     # Don't shutdown the machine after running the test when INTERACTIVE_DEBUG is set
 795     [[ -z $INTERACTIVE_DEBUG ]] && ln -fs $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/testsuite.target.wants/end.service
 796
 797     # make the testsuite the default target
 798     ln -fs testsuite.target $initdir/etc/systemd/system/default.target
 799 }
 800
 801 setup_nspawn_root() {
 802     rm -fr $TESTDIR/nspawn-root
 803     ddebug "cp -ar $initdir $TESTDIR/nspawn-root"
 804     cp -ar $initdir $TESTDIR/nspawn-root
 805     # we don't mount in the nspawn root
 806     rm -f $TESTDIR/nspawn-root/etc/fstab
 807     if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
 808         cp -ar $TESTDIR/nspawn-root $TESTDIR/unprivileged-nspawn-root
 809     fi
 810 }
 811
 812 setup_basic_dirs() {
 813     mkdir -p $initdir/run
 814     mkdir -p $initdir/etc/systemd/system
 815     mkdir -p $initdir/var/log/journal
 816
 817     for d in usr/bin usr/sbin bin etc lib "$libdir" sbin tmp usr var var/log dev proc sys sysroot root run run/lock run/initramfs; do
 818         if [ -L "/$d" ]; then
 819             inst_symlink "/$d"
 820         else
 821             inst_dir "/$d"
 822         fi
 823     done
 824
 825     ln -sfn /run "$initdir/var/run"
 826     ln -sfn /run/lock "$initdir/var/lock"
 827 }
 828
 829 inst_libs() {
 830     local _bin=$1
 831     local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
 832     local _file _line
 833
 834     LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
 835         [[ $_line = 'not a dynamic executable' ]] && break
 836
 837         if [[ $_line =~ $_so_regex ]]; then
 838             _file=${BASH_REMATCH[1]}
 839             [[ -e ${initdir}/$_file ]] && continue
 840             inst_library "$_file"
 841             continue
 842         fi
 843
 844         if [[ $_line =~ not\ found ]]; then
 845             dfatal "Missing a shared library required by $_bin."
 846             dfatal "Run \"ldd $_bin\" to find out what it is."
 847             dfatal "$_line"
 848             dfatal "dracut cannot create an initrd."
 849             exit 1
 850         fi
 851     done
 852 }
 853
 854 import_testdir() {
 855     [[ -e $STATEFILE ]] && . $STATEFILE
 856     if [[ ! -d "$TESTDIR" ]]; then
 857         if [[ -z "$TESTDIR" ]]; then
 858             TESTDIR=$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)
 859         else
 860             mkdir -p "$TESTDIR"
 861         fi
 862
 863         echo "TESTDIR=\"$TESTDIR\"" > $STATEFILE
 864         export TESTDIR
 865     fi
 866 }
 867
 868 import_initdir() {
 869     initdir=$TESTDIR/root
 870     export initdir
 871 }
 872
 873 ## @brief Converts numeric logging level to the first letter of level name.
 874 #
 875 # @param lvl Numeric logging level in range from 1 to 6.
 876 # @retval 1 if @a lvl is out of range.
 877 # @retval 0 if @a lvl is correct.
 878 # @result Echoes first letter of level name.
 879 _lvl2char() {
 880     case "$1" in
 881         1) echo F;;
 882         2) echo E;;
 883         3) echo W;;
 884         4) echo I;;
 885         5) echo D;;
 886         6) echo T;;
 887         *) return 1;;
 888     esac
 889 }
 890
 891 ## @brief Internal helper function for _do_dlog()
 892 #
 893 # @param lvl Numeric logging level.
 894 # @param msg Message.
 895 # @retval 0 It's always returned, even if logging failed.
 896 #
 897 # @note This function is not supposed to be called manually. Please use
 898 # dtrace(), ddebug(), or others instead which wrap this one.
 899 #
 900 # This function calls _do_dlog() either with parameter msg, or if
 901 # none is given, it will read standard input and will use every line as
 902 # a message.
 903 #
 904 # This enables:
 905 # dwarn "This is a warning"
 906 # echo "This is a warning" | dwarn
 907 LOG_LEVEL=${LOG_LEVEL:-4}
 908
 909 dlog() {
 910     [ -z "$LOG_LEVEL" ] && return 0
 911     [ $1 -le $LOG_LEVEL ] || return 0
 912     local lvl="$1"; shift
 913     local lvlc=$(_lvl2char "$lvl") || return 0
 914
 915     if [ $# -ge 1 ]; then
 916         echo "$lvlc: $*"
 917     else
 918         while read line; do
 919             echo "$lvlc: " "$line"
 920         done
 921     fi
 922 }
 923
 924 ## @brief Logs message at TRACE level (6)
 925 #
 926 # @param msg Message.
 927 # @retval 0 It's always returned, even if logging failed.
 928 dtrace() {
 929     set +x
 930     dlog 6 "$@"
 931     [ -n "$debug" ] && set -x || :
 932 }
 933
 934 ## @brief Logs message at DEBUG level (5)
 935 #
 936 # @param msg Message.
 937 # @retval 0 It's always returned, even if logging failed.
 938 ddebug() {
 939 #    set +x
 940     dlog 5 "$@"
 941 #    [ -n "$debug" ] && set -x || :
 942 }
 943
 944 ## @brief Logs message at INFO level (4)
 945 #
 946 # @param msg Message.
 947 # @retval 0 It's always returned, even if logging failed.
 948 dinfo() {
 949     set +x
 950     dlog 4 "$@"
 951     [ -n "$debug" ] && set -x || :
 952 }
 953
 954 ## @brief Logs message at WARN level (3)
 955 #
 956 # @param msg Message.
 957 # @retval 0 It's always returned, even if logging failed.
 958 dwarn() {
 959     set +x
 960     dlog 3 "$@"
 961     [ -n "$debug" ] && set -x || :
 962 }
 963
 964 ## @brief Logs message at ERROR level (2)
 965 #
 966 # @param msg Message.
 967 # @retval 0 It's always returned, even if logging failed.
 968 derror() {
 969 #    set +x
 970     dlog 2 "$@"
 971 #    [ -n "$debug" ] && set -x || :
 972 }
 973
 974 ## @brief Logs message at FATAL level (1)
 975 #
 976 # @param msg Message.
 977 # @retval 0 It's always returned, even if logging failed.
 978 dfatal() {
 979     set +x
 980     dlog 1 "$@"
 981     [ -n "$debug" ] && set -x || :
 982 }
 983
 984
 985 # Generic substring function.  If $2 is in $1, return 0.
 986 strstr() { [ "${1#*$2*}" != "$1" ]; }
 987
 988 # normalize_path <path>
 989 # Prints the normalized path, where it removes any duplicated
 990 # and trailing slashes.
 991 # Example:
 992 # $ normalize_path ///test/test//
 993 # /test/test
 994 normalize_path() {
 995     shopt -q -s extglob
 996     set -- "${1//+(\/)//}"
 997     shopt -q -u extglob
 998     echo "${1%/}"
 999 }
1000
1001 # convert_abs_rel <from> <to>
1002 # Prints the relative path, when creating a symlink to <to> from <from>.
1003 # Example:
1004 # $ convert_abs_rel /usr/bin/test /bin/test-2
1005 # ../../bin/test-2
1006 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
1007 convert_abs_rel() {
1008     local __current __absolute __abssize __cursize __newpath
1009     local -i __i __level
1010
1011     set -- "$(normalize_path "$1")" "$(normalize_path "$2")"
1012
1013     # corner case #1 - self looping link
1014     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
1015
1016     # corner case #2 - own dir link
1017     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
1018
1019     IFS="/" __current=($1)
1020     IFS="/" __absolute=($2)
1021
1022     __abssize=${#__absolute[@]}
1023     __cursize=${#__current[@]}
1024
1025     while [[ ${__absolute[__level]} == ${__current[__level]} ]]
1026     do
1027         (( __level++ ))
1028         if (( __level > __abssize || __level > __cursize ))
1029         then
1030             break
1031         fi
1032     done
1033
1034     for ((__i = __level; __i < __cursize-1; __i++))
1035     do
1036         if ((__i > __level))
1037         then
1038             __newpath=$__newpath"/"
1039         fi
1040         __newpath=$__newpath".."
1041     done
1042
1043     for ((__i = __level; __i < __abssize; __i++))
1044     do
1045         if [[ -n $__newpath ]]
1046         then
1047             __newpath=$__newpath"/"
1048         fi
1049         __newpath=$__newpath${__absolute[__i]}
1050     done
1051
1052     echo "$__newpath"
1053 }
1054
1055
1056 # Install a directory, keeping symlinks as on the original system.
1057 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
1058 # will create ${initdir}/lib64, ${initdir}/lib64/file,
1059 # and a symlink ${initdir}/lib -> lib64.
1060 inst_dir() {
1061     [[ -e ${initdir}/"$1" ]] && return 0  # already there
1062
1063     local _dir="$1" _part="${1%/*}" _file
1064     while [[ "$_part" != "${_part%/*}" ]] && ! [[ -e "${initdir}/${_part}" ]]; do
1065         _dir="$_part $_dir"
1066         _part=${_part%/*}
1067     done
1068
1069     # iterate over parent directories
1070     for _file in $_dir; do
1071         [[ -e "${initdir}/$_file" ]] && continue
1072         if [[ -L $_file ]]; then
1073             inst_symlink "$_file"
1074         else
1075             # create directory
1076             mkdir -m 0755 -p "${initdir}/$_file" || return 1
1077             [[ -e "$_file" ]] && chmod --reference="$_file" "${initdir}/$_file"
1078             chmod u+w "${initdir}/$_file"
1079         fi
1080     done
1081 }
1082
1083 # $1 = file to copy to ramdisk
1084 # $2 (optional) Name for the file on the ramdisk
1085 # Location of the image dir is assumed to be $initdir
1086 # We never overwrite the target if it exists.
1087 inst_simple() {
1088     [[ -f "$1" ]] || return 1
1089     strstr "$1" "/" || return 1
1090
1091     local _src=$1 target="${2:-$1}"
1092     if ! [[ -d ${initdir}/$target ]]; then
1093         [[ -e ${initdir}/$target ]] && return 0
1094         [[ -L ${initdir}/$target ]] && return 0
1095         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
1096     fi
1097     # install checksum files also
1098     if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
1099         inst "${_src%/*}/.${_src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
1100     fi
1101     ddebug "Installing $_src"
1102     cp --sparse=always -pfL "$_src" "${initdir}/$target"
1103 }
1104
1105 # find symlinks linked to given library file
1106 # $1 = library file
1107 # Function searches for symlinks by stripping version numbers appended to
1108 # library filename, checks if it points to the same target and finally
1109 # prints the list of symlinks to stdout.
1110 #
1111 # Example:
1112 # rev_lib_symlinks libfoo.so.8.1
1113 # output: libfoo.so.8 libfoo.so
1114 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
1115 rev_lib_symlinks() {
1116     [[ ! $1 ]] && return 0
1117
1118     local fn="$1" orig="$(readlink -f "$1")" links=''
1119
1120     [[ ${fn} =~ .*\.so\..* ]] || return 1
1121
1122     until [[ ${fn##*.} == so ]]; do
1123         fn="${fn%.*}"
1124         [[ -L ${fn} && $(readlink -f "${fn}") == ${orig} ]] && links+=" ${fn}"
1125     done
1126
1127     echo "${links}"
1128 }
1129
1130 # Same as above, but specialized to handle dynamic libraries.
1131 # It handles making symlinks according to how the original library
1132 # is referenced.
1133 inst_library() {
1134     local _src="$1" _dest=${2:-$1} _lib _reallib _symlink
1135     strstr "$1" "/" || return 1
1136     [[ -e $initdir/$_dest ]] && return 0
1137     if [[ -L $_src ]]; then
1138         # install checksum files also
1139         if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
1140             inst "${_src%/*}/.${_src##*/}.hmac" "${_dest%/*}/.${_dest##*/}.hmac"
1141         fi
1142         _reallib=$(readlink -f "$_src")
1143         inst_simple "$_reallib" "$_reallib"
1144         inst_dir "${_dest%/*}"
1145         [[ -d "${_dest%/*}" ]] && _dest=$(readlink -f "${_dest%/*}")/${_dest##*/}
1146         ln -sfn $(convert_abs_rel "${_dest}" "${_reallib}") "${initdir}/${_dest}"
1147     else
1148         inst_simple "$_src" "$_dest"
1149     fi
1150
1151     # Create additional symlinks.  See rev_symlinks description.
1152     for _symlink in $(rev_lib_symlinks $_src) $(rev_lib_symlinks $_reallib); do
1153         [[ -e $initdir/$_symlink ]] || {
1154             ddebug "Creating extra symlink: $_symlink"
1155             inst_symlink $_symlink
1156         }
1157     done
1158 }
1159
1160 # find a binary.  If we were not passed the full path directly,
1161 # search in the usual places to find the binary.
1162 find_binary() {
1163     if [[ -z ${1##/*} ]]; then
1164         if [[ -x $1 ]] || { strstr "$1" ".so" && ldd $1 &>/dev/null; };  then
1165             echo $1
1166             return 0
1167         fi
1168     fi
1169
1170     type -P $1
1171 }
1172
1173 # Same as above, but specialized to install binary executables.
1174 # Install binary executable, and all shared library dependencies, if any.
1175 inst_binary() {
1176     local _bin _target
1177     _bin=$(find_binary "$1") || return 1
1178     _target=${2:-$_bin}
1179     [[ -e $initdir/$_target ]] && return 0
1180     [[ -L $_bin ]] && inst_symlink $_bin $_target && return 0
1181     local _file _line
1182     local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
1183     # I love bash!
1184     LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
1185         [[ $_line = 'not a dynamic executable' ]] && break
1186
1187         if [[ $_line =~ $_so_regex ]]; then
1188             _file=${BASH_REMATCH[1]}
1189             [[ -e ${initdir}/$_file ]] && continue
1190             inst_library "$_file"
1191             continue
1192         fi
1193
1194         if [[ $_line =~ not\ found ]]; then
1195             dfatal "Missing a shared library required by $_bin."
1196             dfatal "Run \"ldd $_bin\" to find out what it is."
1197             dfatal "$_line"
1198             dfatal "dracut cannot create an initrd."
1199             exit 1
1200         fi
1201     done
1202     inst_simple "$_bin" "$_target"
1203 }
1204
1205 # same as above, except for shell scripts.
1206 # If your shell script does not start with shebang, it is not a shell script.
1207 inst_script() {
1208     local _bin
1209     _bin=$(find_binary "$1") || return 1
1210     shift
1211     local _line _shebang_regex
1212     read -r -n 80 _line <"$_bin"
1213     # If debug is set, clean unprintable chars to prevent messing up the term
1214     [[ $debug ]] && _line=$(echo -n "$_line" | tr -c -d '[:print:][:space:]')
1215     _shebang_regex='(#! *)(/[^ ]+).*'
1216     [[ $_line =~ $_shebang_regex ]] || return 1
1217     inst "${BASH_REMATCH[2]}" && inst_simple "$_bin" "$@"
1218 }
1219
1220 # same as above, but specialized for symlinks
1221 inst_symlink() {
1222     local _src=$1 _target=${2:-$1} _realsrc
1223     strstr "$1" "/" || return 1
1224     [[ -L $1 ]] || return 1
1225     [[ -L $initdir/$_target ]] && return 0
1226     _realsrc=$(readlink -f "$_src")
1227     if ! [[ -e $initdir/$_realsrc ]]; then
1228         if [[ -d $_realsrc ]]; then
1229             inst_dir "$_realsrc"
1230         else
1231             inst "$_realsrc"
1232         fi
1233     fi
1234     [[ ! -e $initdir/${_target%/*} ]] && inst_dir "${_target%/*}"
1235     [[ -d ${_target%/*} ]] && _target=$(readlink -f ${_target%/*})/${_target##*/}
1236     ln -sfn $(convert_abs_rel "${_target}" "${_realsrc}") "$initdir/$_target"
1237 }
1238
1239 # attempt to install any programs specified in a udev rule
1240 inst_rule_programs() {
1241     local _prog _bin
1242
1243     if grep -qE 'PROGRAM==?"[^ "]+' "$1"; then
1244         for _prog in $(grep -E 'PROGRAM==?"[^ "]+' "$1" | sed -r 's/.*PROGRAM==?"([^ "]+).*/\1/'); do
1245             if [ -x /lib/udev/$_prog ]; then
1246                 _bin=/lib/udev/$_prog
1247             else
1248                 _bin=$(find_binary "$_prog") || {
1249                     dinfo "Skipping program $_prog using in udev rule $(basename $1) as it cannot be found"
1250                     continue;
1251                 }
1252             fi
1253
1254             #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
1255             dracut_install "$_bin"
1256         done
1257     fi
1258 }
1259
1260 # udev rules always get installed in the same place, so
1261 # create a function to install them to make life simpler.
1262 inst_rules() {
1263     local _target=/etc/udev/rules.d _rule _found
1264
1265     inst_dir "/lib/udev/rules.d"
1266     inst_dir "$_target"
1267     for _rule in "$@"; do
1268         if [ "${rule#/}" = "$rule" ]; then
1269             for r in /lib/udev/rules.d /etc/udev/rules.d; do
1270                 if [[ -f $r/$_rule ]]; then
1271                     _found="$r/$_rule"
1272                     inst_simple "$_found"
1273                     inst_rule_programs "$_found"
1274                 fi
1275             done
1276         fi
1277         for r in '' ./ $dracutbasedir/rules.d/; do
1278             if [[ -f ${r}$_rule ]]; then
1279                 _found="${r}$_rule"
1280                 inst_simple "$_found" "$_target/${_found##*/}"
1281                 inst_rule_programs "$_found"
1282             fi
1283         done
1284         [[ $_found ]] || dinfo "Skipping udev rule: $_rule"
1285         _found=
1286     done
1287 }
1288
1289 # general purpose installation function
1290 # Same args as above.
1291 inst() {
1292     local _x
1293
1294     case $# in
1295         1) ;;
1296         2) [[ ! $initdir && -d $2 ]] && export initdir=$2
1297             [[ $initdir = $2 ]] && set $1;;
1298         3) [[ -z $initdir ]] && export initdir=$2
1299             set $1 $3;;
1300         *) dfatal "inst only takes 1 or 2 or 3 arguments"
1301             exit 1;;
1302     esac
1303     for _x in inst_symlink inst_script inst_binary inst_simple; do
1304         $_x "$@" && return 0
1305     done
1306     return 1
1307 }
1308
1309 # install any of listed files
1310 #
1311 # If first argument is '-d' and second some destination path, first accessible
1312 # source is installed into this path, otherwise it will installed in the same
1313 # path as source.  If none of listed files was installed, function return 1.
1314 # On first successful installation it returns with 0 status.
1315 #
1316 # Example:
1317 #
1318 # inst_any -d /bin/foo /bin/bar /bin/baz
1319 #
1320 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
1321 # initramfs.
1322 inst_any() {
1323     local to f
1324
1325     [[ $1 = '-d' ]] && to="$2" && shift 2
1326
1327     for f in "$@"; do
1328         if [[ -e $f ]]; then
1329             [[ $to ]] && inst "$f" "$to" && return 0
1330             inst "$f" && return 0
1331         fi
1332     done
1333
1334     return 1
1335 }
1336
1337 # dracut_install [-o ] <file> [<file> ... ]
1338 # Install <file> to the initramfs image
1339 # -o optionally install the <file> and don't fail, if it is not there
1340 dracut_install() {
1341     local _optional=no
1342     if [[ $1 = '-o' ]]; then
1343         _optional=yes
1344         shift
1345     fi
1346     while (($# > 0)); do
1347         if ! inst "$1" ; then
1348             if [[ $_optional = yes ]]; then
1349                 dinfo "Skipping program $1 as it cannot be found and is" \
1350                     "flagged to be optional"
1351             else
1352                 dfatal "Failed to install $1"
1353                 exit 1
1354             fi
1355         fi
1356         shift
1357     done
1358 }
1359
1360 # Install a single kernel module along with any firmware it may require.
1361 # $1 = full path to kernel module to install
1362 install_kmod_with_fw() {
1363     # no need to go further if the module is already installed
1364
1365     [[ -e "${initdir}/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" ]] \
1366         && return 0
1367
1368     [[ -e "$initdir/.kernelmodseen/${1##*/}" ]] && return 0
1369
1370     if [[ $omit_drivers ]]; then
1371         local _kmod=${1##*/}
1372         _kmod=${_kmod%.ko}
1373         _kmod=${_kmod/-/_}
1374         if [[ "$_kmod" =~ $omit_drivers ]]; then
1375             dinfo "Omitting driver $_kmod"
1376             return 1
1377         fi
1378         if [[ "${1##*/lib/modules/$KERNEL_VER/}" =~ $omit_drivers ]]; then
1379             dinfo "Omitting driver $_kmod"
1380             return 1
1381         fi
1382     fi
1383
1384     [ -d "$initdir/.kernelmodseen" ] && \
1385         > "$initdir/.kernelmodseen/${1##*/}"
1386
1387     inst_simple "$1" "/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" \
1388         || return $?
1389
1390     local _modname=${1##*/} _fwdir _found _fw
1391     _modname=${_modname%.ko*}
1392     for _fw in $(modinfo -k $KERNEL_VER -F firmware $1 2>/dev/null); do
1393         _found=''
1394         for _fwdir in $fw_dir; do
1395             if [[ -d $_fwdir && -f $_fwdir/$_fw ]]; then
1396                 inst_simple "$_fwdir/$_fw" "/lib/firmware/$_fw"
1397                 _found=yes
1398             fi
1399         done
1400         if [[ $_found != yes ]]; then
1401             if ! grep -qe "\<${_modname//-/_}\>" /proc/modules; then
1402                 dinfo "Possible missing firmware \"${_fw}\" for kernel module" \
1403                     "\"${_modname}.ko\""
1404             else
1405                 dwarn "Possible missing firmware \"${_fw}\" for kernel module" \
1406                     "\"${_modname}.ko\""
1407             fi
1408         fi
1409     done
1410     return 0
1411 }
1412
1413 # Do something with all the dependencies of a kernel module.
1414 # Note that kernel modules depend on themselves using the technique we use
1415 # $1 = function to call for each dependency we find
1416 #      It will be passed the full path to the found kernel module
1417 # $2 = module to get dependencies for
1418 # rest of args = arguments to modprobe
1419 # _fderr specifies FD passed from surrounding scope
1420 for_each_kmod_dep() {
1421     local _func=$1 _kmod=$2 _cmd _modpath _options _found=0
1422     shift 2
1423     modprobe "$@" --ignore-install --show-depends $_kmod 2>&${_fderr} | (
1424         while read _cmd _modpath _options; do
1425             [[ $_cmd = insmod ]] || continue
1426             $_func ${_modpath} || exit $?
1427             _found=1
1428         done
1429         [[ $_found -eq 0 ]] && exit 1
1430         exit 0
1431     )
1432 }
1433
1434 # filter kernel modules to install certain modules that meet specific
1435 # requirements.
1436 # $1 = search only in subdirectory of /kernel/$1
1437 # $2 = function to call with module name to filter.
1438 #      This function will be passed the full path to the module to test.
1439 # The behavior of this function can vary depending on whether $hostonly is set.
1440 # If it is, we will only look at modules that are already in memory.
1441 # If it is not, we will look at all kernel modules
1442 # This function returns the full filenames of modules that match $1
1443 filter_kernel_modules_by_path () (
1444     local _modname _filtercmd
1445     if ! [[ $hostonly ]]; then
1446         _filtercmd='find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra"'
1447         _filtercmd+=' "$KERNEL_MODS/weak-updates" -name "*.ko" -o -name "*.ko.gz"'
1448         _filtercmd+=' -o -name "*.ko.xz"'
1449         _filtercmd+=' 2>/dev/null'
1450     else
1451         _filtercmd='cut -d " " -f 1 </proc/modules|xargs modinfo -F filename '
1452         _filtercmd+='-k $KERNEL_VER 2>/dev/null'
1453     fi
1454     for _modname in $(eval $_filtercmd); do
1455         case $_modname in
1456             *.ko) "$2" "$_modname" && echo "$_modname";;
1457             *.ko.gz) gzip -dc "$_modname" > $initdir/$$.ko
1458                 $2 $initdir/$$.ko && echo "$_modname"
1459                 rm -f $initdir/$$.ko
1460                 ;;
1461             *.ko.xz) xz -dc "$_modname" > $initdir/$$.ko
1462                 $2 $initdir/$$.ko && echo "$_modname"
1463                 rm -f $initdir/$$.ko
1464                 ;;
1465         esac
1466     done
1467 )
1468 find_kernel_modules_by_path () (
1469     if ! [[ $hostonly ]]; then
1470         find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra" "$KERNEL_MODS/weak-updates" \
1471           -name "*.ko" -o -name "*.ko.gz" -o -name "*.ko.xz" 2>/dev/null
1472     else
1473         cut -d " " -f 1 </proc/modules \
1474         | xargs modinfo -F filename -k $KERNEL_VER 2>/dev/null
1475     fi
1476 )
1477
1478 filter_kernel_modules () {
1479     filter_kernel_modules_by_path  drivers  "$1"
1480 }
1481
1482 find_kernel_modules () {
1483     find_kernel_modules_by_path  drivers
1484 }
1485
1486 # instmods [-c] <kernel module> [<kernel module> ... ]
1487 # instmods [-c] <kernel subsystem>
1488 # install kernel modules along with all their dependencies.
1489 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
1490 instmods() {
1491     [[ $no_kernel = yes ]] && return
1492     # called [sub]functions inherit _fderr
1493     local _fderr=9
1494     local _check=no
1495     if [[ $1 = '-c' ]]; then
1496         _check=yes
1497         shift
1498     fi
1499
1500     function inst1mod() {
1501         local _ret=0 _mod="$1"
1502         case $_mod in
1503             =*)
1504                 if [ -f $KERNEL_MODS/modules.${_mod#=} ]; then
1505                     ( [[ "$_mpargs" ]] && echo $_mpargs
1506                       cat "${KERNEL_MODS}/modules.${_mod#=}" ) \
1507                     | instmods
1508                 else
1509                     ( [[ "$_mpargs" ]] && echo $_mpargs
1510                       find "$KERNEL_MODS" -path "*/${_mod#=}/*" -printf '%f\n' ) \
1511                     | instmods
1512                 fi
1513                 ;;
1514             --*) _mpargs+=" $_mod" ;;
1515             i2o_scsi) return ;; # Do not load this diagnostic-only module
1516             *)
1517                 _mod=${_mod##*/}
1518                 # if we are already installed, skip this module and go on
1519                 # to the next one.
1520                 [[ -f "$initdir/.kernelmodseen/${_mod%.ko}.ko" ]] && return
1521
1522                 if [[ $omit_drivers ]] && [[ "$1" =~ $omit_drivers ]]; then
1523                     dinfo "Omitting driver ${_mod##$KERNEL_MODS}"
1524                     return
1525                 fi
1526                 # If we are building a host-specific initramfs and this
1527                 # module is not already loaded, move on to the next one.
1528                 [[ $hostonly ]] && ! grep -qe "\<${_mod//-/_}\>" /proc/modules \
1529                     && ! echo $add_drivers | grep -qe "\<${_mod}\>" \
1530                     && return
1531
1532                 # We use '-d' option in modprobe only if modules prefix path
1533                 # differs from default '/'.  This allows us to use Dracut with
1534                 # old version of modprobe which doesn't have '-d' option.
1535                 local _moddirname=${KERNEL_MODS%%/lib/modules/*}
1536                 [[ -n ${_moddirname} ]] && _moddirname="-d ${_moddirname}/"
1537
1538                 # ok, load the module, all its dependencies, and any firmware
1539                 # it may require
1540                 for_each_kmod_dep install_kmod_with_fw $_mod \
1541                     --set-version $KERNEL_VER ${_moddirname} $_mpargs
1542                 ((_ret+=$?))
1543                 ;;
1544         esac
1545         return $_ret
1546     }
1547
1548     function instmods_1() {
1549         local _mod _mpargs
1550         if (($# == 0)); then  # filenames from stdin
1551             while read _mod; do
1552                 inst1mod "${_mod%.ko*}" || {
1553                     if [ "$_check" = "yes" ]; then
1554                         dfatal "Failed to install $_mod"
1555                         return 1
1556                     fi
1557                 }
1558             done
1559         fi
1560         while (($# > 0)); do  # filenames as arguments
1561             inst1mod ${1%.ko*} || {
1562                 if [ "$_check" = "yes" ]; then
1563                     dfatal "Failed to install $1"
1564                     return 1
1565                 fi
1566             }
1567             shift
1568         done
1569         return 0
1570     }
1571
1572     local _ret _filter_not_found='FATAL: Module .* not found.'
1573     set -o pipefail
1574     # Capture all stderr from modprobe to _fderr. We could use {var}>...
1575     # redirections, but that would make dracut require bash4 at least.
1576     eval "( instmods_1 \"\$@\" ) ${_fderr}>&1" \
1577     | while read line; do [[ "$line" =~ $_filter_not_found ]] && echo $line || echo $line >&2 ;done | derror
1578     _ret=$?
1579     set +o pipefail
1580     return $_ret
1581 }
1582
1583 # inst_libdir_file [-n <pattern>] <file> [<file>...]
1584 # Install a <file> located on a lib directory to the initramfs image
1585 # -n <pattern> install non-matching files
1586 inst_libdir_file() {
1587     if [[ "$1" == "-n" ]]; then
1588         local _pattern=$1
1589         shift 2
1590         for _dir in $libdirs; do
1591             for _i in "$@"; do
1592                 for _f in "$_dir"/$_i; do
1593                     [[ "$_i" =~ $_pattern ]] || continue
1594                     [[ -e "$_i" ]] && dracut_install "$_i"
1595                 done
1596             done
1597         done
1598     else
1599         for _dir in $libdirs; do
1600             for _i in "$@"; do
1601                 for _f in "$_dir"/$_i; do
1602                     [[ -e "$_f" ]] && dracut_install "$_f"
1603                 done
1604             done
1605         done
1606     fi
1607 }
1608
1609 setup_suse() {
1610     ln -fs ../usr/bin/systemctl $initdir/bin/
1611     ln -fs ../usr/lib/systemd $initdir/lib/
1612     inst_simple "/usr/lib/systemd/system/haveged.service"
1613 }
1614
1615 # can be overridden in specific test
1616 test_cleanup() {
1617     umount $TESTDIR/root 2>/dev/null || true
1618     [[ $LOOPDEV ]] && losetup -d $LOOPDEV || true
1619     return 0
1620 }
1621
1622 test_run() {
1623     if [ -z "$TEST_NO_QEMU" ]; then
1624         if run_qemu; then
1625             check_result_qemu || return 1
1626         else
1627             dwarn "can't run QEMU, skipping"
1628         fi
1629     fi
1630     if [ -z "$TEST_NO_NSPAWN" ]; then
1631         if run_nspawn "nspawn-root"; then
1632             check_result_nspawn "nspawn-root" || return 1
1633         else
1634             dwarn "can't run systemd-nspawn, skipping"
1635         fi
1636
1637         if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
1638             if NSPAWN_ARGUMENTS="-U --private-network $NSPAWN_ARGUMENTS" run_nspawn "unprivileged-nspawn-root"; then
1639                 check_result_nspawn "unprivileged-nspawn-root" || return 1
1640             else
1641                 dwarn "can't run systemd-nspawn, skipping"
1642             fi
1643         fi
1644     fi
1645     return 0
1646 }
1647
1648 do_test() {
1649     if [[ $UID != "0" ]]; then
1650         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
1651         exit 0
1652     fi
1653
1654     # Detect lib paths
1655     [[ $libdir ]] || for libdir in /lib64 /lib; do
1656         [[ -d $libdir ]] && libdirs+=" $libdir" && break
1657     done
1658
1659     [[ $usrlibdir ]] || for usrlibdir in /usr/lib64 /usr/lib; do
1660         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
1661     done
1662
1663     mkdir -p "$STATEDIR"
1664
1665     import_testdir
1666     import_initdir
1667
1668     while (($# > 0)); do
1669         case $1 in
1670             --run)
1671                 echo "TEST RUN: $TEST_DESCRIPTION"
1672                 test_run
1673                 ret=$?
1674                 if (( $ret == 0 )); then
1675                     echo "TEST RUN: $TEST_DESCRIPTION [OK]"
1676                 else
1677                     echo "TEST RUN: $TEST_DESCRIPTION [FAILED]"
1678                 fi
1679                 exit $ret;;
1680             --setup)
1681                 echo "TEST SETUP: $TEST_DESCRIPTION"
1682                 test_setup
1683                 ;;
1684             --clean)
1685                 echo "TEST CLEANUP: $TEST_DESCRIPTION"
1686                 test_cleanup
1687                 rm -fr "$TESTDIR"
1688                 rm -f "$STATEFILE"
1689                 ;;
1690             --all)
1691                 ret=0
1692                 echo -n "TEST: $TEST_DESCRIPTION ";
1693                 (
1694                     test_setup && test_run
1695                     ret=$?
1696                     test_cleanup
1697                     rm -fr "$TESTDIR"
1698                     rm -f "$STATEFILE"
1699                     exit $ret
1700                 ) </dev/null >"$TESTLOG" 2>&1 || ret=$?
1701                 if [ $ret -eq 0 ]; then
1702                     rm "$TESTLOG"
1703                     echo "[OK]"
1704                 else
1705                     echo "[FAILED]"
1706                     echo "see $TESTLOG"
1707                 fi
1708                 exit $ret;;
1709             *) break ;;
1710         esac
1711         shift
1712     done
1713 }