tests: redirect the stdout/stderr of journald to a file (under ASan+UBSan)

[thirdparty/systemd.git] / test / test-functions

#!/bin/bash
# -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
# ex: ts=8 sw=4 sts=4 et filetype=sh
PATH=/sbin:/bin:/usr/sbin:/usr/bin
export PATH

LOOKS_LIKE_DEBIAN=$(source /etc/os-release && [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && echo yes || true)
LOOKS_LIKE_ARCH=$(source /etc/os-release && [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && echo yes || true)
LOOKS_LIKE_SUSE=$(source /etc/os-release && [[ " $ID_LIKE " = *" suse "* ]] && echo yes || true)
KERNEL_VER=${KERNEL_VER-$(uname -r)}
KERNEL_MODS="/lib/modules/$KERNEL_VER/"
QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
[[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
EFI_MOUNT="$(bootctl -p 2>/dev/null || echo /boot)"
QEMU_MEM="${QEMU_MEM:-512M}"

if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
    echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
    ROOTLIBDIR=/usr/lib/systemd
fi

PATH_TO_INIT=$ROOTLIBDIR/systemd

BASICTOOLS="test sh bash setsid loadkeys setfont login sulogin gzip sleep echo head tail cat mount umount cryptsetup date dmsetup modprobe sed cmp tee rm true false chmod chown ln xargs"
DEBUGTOOLS="df free ls stty ps ln ip route dmesg dhclient mkdir cp ping dhclient strace less grep id tty touch du sort hostname find vi mv"

STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))"
STATEFILE="$STATEDIR/.testdir"
TESTLOG="$STATEDIR/test.log"

is_built_with_asan() {
    if ! type -P objdump >/dev/null; then
        ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
        return 1
    fi

    # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
    local _asan_calls=$(objdump -dC $BUILD_DIR/systemd-journald | egrep "callq\s+[0-9a-f]+\s+<__asan" -c)
    if (( $_asan_calls < 1000 )); then
        return 1
    else
        return 0
    fi
}

IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)

if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
    STRIP_BINARIES=no
    SKIP_INITRD="${SKIP_INITRD:-yes}"
    PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
    QEMU_MEM="1536M"
    QEMU_SMP=4
fi

function find_qemu_bin() {
    # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
    # Either way, only use this version if we aren't running in KVM, because
    # nested KVM is flaky still.
    if [[ $(systemd-detect-virt -v) != kvm && -z $TEST_NO_KVM ]] ; then
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a kvm qemu-kvm 2>/dev/null | grep '^/' -m1)
    fi

    [ "$ARCH" ] || ARCH=$(uname -m)
    case $ARCH in
    x86_64)
        # QEMU's own build system calls it qemu-system-x86_64
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-x86_64 2>/dev/null | grep '^/' -m1)
        ;;
    i*86)
        # new i386 version of QEMU
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-i386 2>/dev/null | grep '^/' -m1)

        # i386 version of QEMU
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu 2>/dev/null | grep '^/' -m1)
        ;;
    ppc64*)
        [ "$QEMU_BIN" ] || QEMU_BIN=$(which -a qemu-system-$ARCH 2>/dev/null | grep '^/' -m1)
        ;;
    esac

    if [ ! -e "$QEMU_BIN" ]; then
        echo "Could not find a suitable QEMU binary" >&2
        return 1
    fi
}

# Return 0 if QEMU did run (then you must check the result state/logs for actual
# success), or 1 if QEMU is not available.
run_qemu() {
    if [ -f /etc/machine-id ]; then
        read MACHINE_ID < /etc/machine-id
        [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
            && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
        [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
            && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
    fi

    CONSOLE=ttyS0

    if [[ ! "$KERNEL_BIN" ]]; then
        if [[ "$LOOKS_LIKE_ARCH" ]]; then
            KERNEL_BIN=/boot/vmlinuz-linux
        else
            [ "$ARCH" ] || ARCH=$(uname -m)
            case $ARCH in
                ppc64*)
                KERNEL_BIN=/boot/vmlinux-$KERNEL_VER
                CONSOLE=hvc0
                ;;
                *)
                KERNEL_BIN=/boot/vmlinuz-$KERNEL_VER
                ;;
            esac
        fi
    fi

    default_fedora_initrd=/boot/initramfs-${KERNEL_VER}.img
    default_debian_initrd=/boot/initrd.img-${KERNEL_VER}
    default_arch_initrd=/boot/initramfs-linux-fallback.img
    default_suse_initrd=/boot/initrd-${KERNEL_VER}
    if [[ ! "$INITRD" ]]; then
        if [[ -e "$default_fedora_initrd" ]]; then
            INITRD="$default_fedora_initrd"
        elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
            INITRD="$default_debian_initrd"
        elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
            INITRD="$default_arch_initrd"
        elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
            INITRD="$default_suse_initrd"
        fi
    fi

    # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
    # i.e. use the number of online CPUs on the host machine. If the nproc utility
    # is not installed or there's some other error when calling it, fall back
    # to the original value (QEMU_SMP=1).
    if ! [ "$QEMU_SMP" ]; then
        if ! QEMU_SMP=$(nproc); then
            dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
            QEMU_SMP=1
        fi
    fi

    find_qemu_bin || return 1

    local _cgroup_args
    if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
        _cgroup_args="systemd.unified_cgroup_hierarchy=yes"
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
        _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=yes"
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
        _cgroup_args="systemd.unified_cgroup_hierarchy=no systemd.legacy_systemd_cgroup_controller=no"
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
        dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
        exit 1
    fi

    if [[ "$LOOKS_LIKE_SUSE" ]]; then
        PARAMS+="rd.hostonly=0"
    elif [[ "$LOOKS_LIKE_ARCH" ]]; then
        PARAMS+="rw"
    else
        PARAMS+="ro"
    fi

    KERNEL_APPEND="$PARAMS \
root=/dev/sda1 \
raid=noautodetect \
loglevel=2 \
init=$PATH_TO_INIT \
console=$CONSOLE \
selinux=0 \
printk.devkmsg=on \
$_cgroup_args \
$KERNEL_APPEND \
"

    QEMU_OPTIONS="-smp $QEMU_SMP \
-net none \
-m $QEMU_MEM \
-nographic \
-kernel $KERNEL_BIN \
-drive format=raw,cache=unsafe,file=${TESTDIR}/rootdisk.img \
"

    if [[ "$INITRD" && "$SKIP_INITRD" != "yes" ]]; then
        QEMU_OPTIONS="$QEMU_OPTIONS -initrd $INITRD"
    fi

    # Let's use KVM if it is available, but let's avoid using nested KVM as that is still flaky
    if [[ -c /dev/kvm && $(systemd-detect-virt -v) != kvm && -z $TEST_NO_KVM ]] ; then
        QEMU_OPTIONS="$QEMU_OPTIONS -machine accel=kvm -enable-kvm -cpu host"
    fi

    if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
        QEMU_BIN="timeout --foreground $QEMU_TIMEOUT $QEMU_BIN"
    fi
    (set -x; $QEMU_BIN $QEMU_OPTIONS -append "$KERNEL_APPEND")
    rc=$?
    if [ "$rc" = 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
        derror "test timed out after $QEMU_TIMEOUT s"
        TIMED_OUT=1
    else
        [ "$rc" != 0 ] && derror "QEMU failed with exit code $rc"
    fi
    return 0
}

# Return 0 if nspawn did run (then you must check the result state/logs for actual
# success), or 1 if nspawn is not available.
run_nspawn() {
    [[ -d /run/systemd/system ]] || return 1

    local _nspawn_cmd="$BUILD_DIR/systemd-nspawn $NSPAWN_ARGUMENTS --register=no --kill-signal=SIGKILL --directory=$TESTDIR/$1 $PATH_TO_INIT $KERNEL_APPEND"
    if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
        _nspawn_cmd="timeout --foreground $NSPAWN_TIMEOUT $_nspawn_cmd"
    fi

    if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
        dwarn "nspawn doesn't support UNIFIED_CGROUP_HIERARCHY=hybrid, skipping"
        exit
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
        _nspawn_cmd="env UNIFIED_CGROUP_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY $_nspawn_cmd"
    elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
        _nspawn_cmd="env --unset=UNIFIED_CGROUP_HIERARCHY $_nspawn_cmd"
    else
        dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
        exit 1
    fi

    (set -x; $_nspawn_cmd)
    rc=$?
    if [ "$rc" = 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
        derror "test timed out after $NSPAWN_TIMEOUT s"
        TIMED_OUT=1
    else
        [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
    fi
    return 0
}

setup_basic_environment() {
    # create the basic filesystem layout
    setup_basic_dirs

    install_systemd
    install_missing_libraries
    install_config_files
    create_rc_local
    install_basic_tools
    install_libnss
    install_pam
    install_dbus
    install_fonts
    install_keymaps
    install_terminfo
    install_execs
    install_fsck
    install_plymouth
    install_debug_tools
    install_ld_so_conf
    setup_selinux
    strip_binaries
    install_depmod_files
    generate_module_dependencies
    if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
         create_asan_wrapper
    fi
}

setup_selinux() {
    # don't forget KERNEL_APPEND='... selinux=1 ...'
    if [[ "$SETUP_SELINUX" != "yes" ]]; then
        ddebug "Don't setup SELinux"
        return 0
    fi
    ddebug "Setup SELinux"
    local _conf_dir=/etc/selinux
    local _fixfiles_tools="bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles"

    rm -rf $initdir/$_conf_dir
    if ! cp -ar $_conf_dir $initdir/$_conf_dir; then
        dfatal "Failed to copy $_conf_dir"
        exit 1
    fi

    cat <<EOF >$initdir/etc/systemd/system/autorelabel.service
[Unit]
Description=Relabel all filesystems
DefaultDependencies=no
Requires=local-fs.target
Conflicts=shutdown.target
After=local-fs.target
Before=sysinit.target shutdown.target
ConditionSecurity=selinux
ConditionPathExists=|/.autorelabel

[Service]
ExecStart=/bin/sh -x -c 'echo 0 >/sys/fs/selinux/enforce && fixfiles -f -F relabel && rm /.autorelabel && systemctl --force reboot'
Type=oneshot
TimeoutSec=0
RemainAfterExit=yes
EOF

    touch $initdir/.autorelabel
    mkdir -p $initdir/etc/systemd/system/basic.target.wants
    ln -fs autorelabel.service $initdir/etc/systemd/system/basic.target.wants/autorelabel.service

    dracut_install $_fixfiles_tools
    dracut_install fixfiles
    dracut_install sestatus
}

install_valgrind() {
    if ! type -p valgrind; then
        dfatal "Failed to install valgrind"
        exit 1
    fi

    local _valgrind_bins=$(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')
    dracut_install $_valgrind_bins

    local _valgrind_libs=$(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')
    dracut_install $_valgrind_libs

    local _valgrind_dbg_and_supp=$(
        strace -e open valgrind /bin/true 2>&1 >/dev/null |
        perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
    )
    dracut_install $_valgrind_dbg_and_supp
}

create_valgrind_wrapper() {
    local _valgrind_wrapper=$initdir/$ROOTLIBDIR/systemd-under-valgrind
    ddebug "Create $_valgrind_wrapper"
    cat >$_valgrind_wrapper <<EOF
#!/bin/bash

mount -t proc proc /proc
exec valgrind --leak-check=full --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
EOF
    chmod 0755 $_valgrind_wrapper
}

create_asan_wrapper() {
    local _asan_wrapper=$initdir/$ROOTLIBDIR/systemd-under-asan
    ddebug "Create $_asan_wrapper"
    cat >$_asan_wrapper <<EOF
#!/bin/bash

set -x

DEFAULT_ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
DEFAULT_UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS"

mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -o remount,rw /

PATH_TO_ASAN=\$(find / -name '*libasan*' | sed 1q)
if [[ "\$PATH_TO_ASAN" ]]; then
  # A lot of services (most notably dbus) won't start without preloading libasan
  # See https://github.com/systemd/systemd/issues/5004
  DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=\$PATH_TO_ASAN"
fi
echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf

# ASAN and syscall filters aren't compatible with each other.
find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'

# The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
# But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
mkdir -p "\$JOURNALD_CONF_DIR"
printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"

# Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
# Let's try to catch them by redirecting stderr (and stdout just in case) to a file
# See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"

# 90s isn't enough for some services to finish when literally everything is run
# under ASan+UBSan in containers, which, in turn, are run in VMs.
# Let's limit which environments such services should be executed in.
mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf

# The 'mount' utility doesn't behave well under libasan, causing unexpected
# fails during boot and subsequent test results check:
# bash-5.0# mount -o remount,rw -v /
# mount: /dev/sda1 mounted on /.
# bash-5.0# echo \$?
# 1
# Let's workaround this by clearing the previously set LD_PRELOAD env variable,
# so the libasan library is not loaded for this particular service
REMOUNTFS_CONF_DIR=/etc/systemd/system/systemd-remount-fs.service.d
mkdir -p "\$REMOUNTFS_CONF_DIR"
printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$REMOUNTFS_CONF_DIR/env.conf"

export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
exec  $ROOTLIBDIR/systemd "\$@"
EOF

    chmod 0755 $_asan_wrapper
}

create_strace_wrapper() {
    local _strace_wrapper=$initdir/$ROOTLIBDIR/systemd-under-strace
    ddebug "Create $_strace_wrapper"
    cat >$_strace_wrapper <<EOF
#!/bin/bash

exec strace -D -o /strace.out $ROOTLIBDIR/systemd "\$@"
EOF
    chmod 0755 $_strace_wrapper
}

install_fsck() {
    dracut_install /sbin/fsck*
    dracut_install -o /bin/fsck*

    # fskc.reiserfs calls reiserfsck. so, install it
    dracut_install -o reiserfsck
}

install_dmevent() {
    instmods dm_crypt =crypto
    type -P dmeventd >/dev/null && dracut_install dmeventd
    inst_libdir_file "libdevmapper-event.so*"
    if [[ "$LOOKS_LIKE_DEBIAN" ]]; then
        # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
        # and since buster/bionic 95-dm-notify.rules
        # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
        inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
    else
        inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
    fi
}

install_systemd() {
    # install compiled files
    local _ninja_bin=$(type -P ninja || type -P ninja-build)
    if [[ -z "$_ninja_bin" ]]; then
        dfatal "ninja was not found"
        exit 1
    fi
    (set -x; DESTDIR=$initdir "$_ninja_bin" -C $BUILD_DIR install)
    # remove unneeded documentation
    rm -fr $initdir/usr/share/{man,doc}
    # we strip binaries since debug symbols increase binaries size a lot
    # and it could fill the available space
    strip_binaries

   [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse

    # enable debug logging in PID1
    echo LogLevel=debug >> $initdir/etc/systemd/system.conf
}

get_ldpath() {
    local _bin="$1"
    objdump -p "$_bin" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :
}

install_missing_libraries() {
    # install possible missing libraries
    for i in $initdir{,/usr}/{sbin,bin}/* $initdir{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
        LD_LIBRARY_PATH=$(get_ldpath $i) inst_libs $i
    done
}

create_empty_image() {
    local _size=500
    if [[ "$STRIP_BINARIES" = "no" ]]; then
        _size=$((2*_size))
    fi
    rm -f "$TESTDIR/rootdisk.img"
    # Create the blank file to use as a root filesystem
    dd if=/dev/null of="$TESTDIR/rootdisk.img" bs=1M seek="$_size"
    LOOPDEV=$(losetup --show -P -f $TESTDIR/rootdisk.img)
    [ -b "$LOOPDEV" ] || return 1
    echo "LOOPDEV=$LOOPDEV" >> $STATEFILE
    sfdisk "$LOOPDEV" <<EOF
,$((_size-50))M
,
EOF

    udevadm settle

    local _label="-L systemd"
    # mkfs.reiserfs doesn't know -L. so, use --label instead
    [[ "$FSTYPE" == "reiserfs" ]] && _label="--label systemd"
    if ! mkfs -t "${FSTYPE}" ${_label} "${LOOPDEV}p1" -q; then
        dfatal "Failed to mkfs -t ${FSTYPE}"
        exit 1
    fi
}

check_asan_reports() {
    local ret=0
    local root="$1"

    if [[ "$IS_BUILT_WITH_ASAN" = "yes" ]]; then
        ls -l "$root"
        if [[ -e "$root/systemd.asan.log.1" ]]; then
            cat "$root/systemd.asan.log.1"
            ret=$(($ret+1))
        fi

        journald_report=$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)
        if [[ ! -z "$journald_report" ]]; then
            printf "%s\n" "$journald_report"
            cat "$root/systemd-journald.out" || true
            ret=$(($ret+1))
        fi

        pids=$(
            "$BUILD_DIR/journalctl" -D "$root/var/log/journal" | perl -alne '
                 BEGIN {
                     %services_to_ignore = (
                         "dbus-daemon" => undef,
                     );
                 }
                 print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
        )
        if [[ ! -z "$pids" ]]; then
            ret=$(($ret+1))
            for pid in $pids; do
                "$BUILD_DIR/journalctl" -D "$root/var/log/journal" _PID=$pid --no-pager
            done
        fi
    fi

    return $ret
}

check_result_nspawn() {
    local ret=1
    local journald_report=""
    local pids=""
    [[ -e $TESTDIR/$1/testok ]] && ret=0
    [[ -f $TESTDIR/$1/failed ]] && cp -a $TESTDIR/$1/failed $TESTDIR
    cp -a $TESTDIR/$1/var/log/journal $TESTDIR
    [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
    ls -l $TESTDIR/journal/*/*.journal
    test -s $TESTDIR/failed && ret=$(($ret+1))
    [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
    check_asan_reports "$TESTDIR/$1" || ret=$(($ret+1))
    return $ret
}

# can be overridden in specific test
check_result_qemu() {
    local ret=1
    mkdir -p $TESTDIR/root
    mount ${LOOPDEV}p1 $TESTDIR/root
    [[ -e $TESTDIR/root/testok ]] && ret=0
    [[ -f $TESTDIR/root/failed ]] && cp -a $TESTDIR/root/failed $TESTDIR
    cp -a $TESTDIR/root/var/log/journal $TESTDIR
    check_asan_reports "$TESTDIR/root" || ret=$(($ret+1))
    umount $TESTDIR/root
    [[ -f $TESTDIR/failed ]] && cat $TESTDIR/failed
    ls -l $TESTDIR/journal/*/*.journal
    test -s $TESTDIR/failed && ret=$(($ret+1))
    [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
    return $ret
}

strip_binaries() {
    if [[ "$STRIP_BINARIES" = "no" ]]; then
        ddebug "Don't strip binaries"
        return 0
    fi
    ddebug "Strip binaries"
    find "$initdir" -executable -not -path '*/lib/modules/*.ko' -type f | \
        xargs strip --strip-unneeded |& \
        grep -v 'file format not recognized' | \
        ddebug
}

create_rc_local() {
    mkdir -p $initdir/etc/rc.d
    cat >$initdir/etc/rc.d/rc.local <<EOF
#!/bin/bash
exit 0
EOF
    chmod 0755 $initdir/etc/rc.d/rc.local
}

install_execs() {
    ddebug "install any Execs from the service files"
    (
    export PKG_CONFIG_PATH=$BUILD_DIR/src/core/
    systemdsystemunitdir=$(pkg-config --variable=systemdsystemunitdir systemd)
    systemduserunitdir=$(pkg-config --variable=systemduserunitdir systemd)
    sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' $initdir/{$systemdsystemunitdir,$systemduserunitdir}/*.service \
         | sort -u | while read i; do
         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
         # also, plymouth is pulled in by rescue.service, but even there the exit code
         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
         inst $i || [ "${i%.local}" != "$i" ] || [ "${i%systemd-update-done}" != "$i" ] || [ "/bin/plymouth" == "$i" ]
     done
    )
}

generate_module_dependencies() {
    if [[ -d $initdir/lib/modules/$KERNEL_VER ]] && \
        ! depmod -a -b "$initdir" $KERNEL_VER; then
            dfatal "\"depmod -a $KERNEL_VER\" failed."
            exit 1
    fi
}

install_depmod_files() {
    inst /lib/modules/$KERNEL_VER/modules.order
    inst /lib/modules/$KERNEL_VER/modules.builtin
}

install_plymouth() {
    # install plymouth, if found... else remove plymouth service files
    # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
    #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
    #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
    #         dracut_install plymouth plymouthd
    # else
        rm -f $initdir/{usr/lib,etc}/systemd/system/plymouth* $initdir/{usr/lib,etc}/systemd/system/*/plymouth*
    # fi
}

install_ld_so_conf() {
    cp -a /etc/ld.so.conf* $initdir/etc
    ldconfig -r "$initdir"
}

install_config_files() {
    inst /etc/sysconfig/init || true
    inst /etc/passwd
    inst /etc/shadow
    inst /etc/login.defs
    inst /etc/group
    inst /etc/shells
    inst /etc/nsswitch.conf
    inst /etc/pam.conf || true
    inst /etc/securetty || true
    inst /etc/os-release
    inst /etc/localtime
    # we want an empty environment
    > $initdir/etc/environment
    > $initdir/etc/machine-id
    # set the hostname
    echo systemd-testsuite > $initdir/etc/hostname
    # fstab
    if [[ "$LOOKS_LIKE_SUSE" ]]; then
       ROOTMOUNT="/dev/sda1           /       ${FSTYPE}    rw 0 1"
    else
       ROOTMOUNT="LABEL=systemd           /       ${FSTYPE}    rw 0 1"
    fi

    cat >$initdir/etc/fstab <<EOF
$ROOTMOUNT
EOF
}

install_basic_tools() {
    [[ $BASICTOOLS ]] && dracut_install $BASICTOOLS
    dracut_install -o sushell
    # in Debian ldconfig is just a shell script wrapper around ldconfig.real
    dracut_install -o ldconfig.real
}

install_debug_tools() {
    [[ $DEBUGTOOLS ]] && dracut_install $DEBUGTOOLS

    if [[ $INTERACTIVE_DEBUG ]]; then
        # Set default TERM from vt220 to linux, so at least basic key shortcuts work
        local _getty_override="$initdir/etc/systemd/system/serial-getty@.service.d"
        mkdir -p "$_getty_override"
        echo -e "[Service]\nEnvironment=TERM=linux" > "$_getty_override/default-TERM.conf"

        cat > "$initdir/etc/motd" << EOF
To adjust the terminal size use:
    export COLUMNS=xx
    export LINES=yy
or
    stty cols xx rows yy
EOF
    fi
}

install_libnss() {
    # install libnss_files for login
    NSS_LIBS=$(LD_DEBUG=files getent passwd 2>&1 >/dev/null |sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
    dracut_install $NSS_LIBS
}

install_dbus() {
    inst $ROOTLIBDIR/system/dbus.socket

    # Fedora rawhide replaced dbus.service with dbus-daemon.service
    if [ -f $ROOTLIBDIR/system/dbus-daemon.service ]; then
        inst $ROOTLIBDIR/system/dbus-daemon.service
        # Alias symlink
        inst_symlink /etc/systemd/system/dbus.service
    else
        inst $ROOTLIBDIR/system/dbus.service
    fi
    # Newer Fedora versions use dbus-broker by default. Let's install it is available.
    [ -f /usr/bin/dbus-broker ] && inst /usr/bin/dbus-broker
    [ -f /usr/bin/dbus-broker-launch ] && inst /usr/bin/dbus-broker-launch

    find \
        /etc/dbus-1 /usr/share/dbus-1 -xtype f \
        | while read file; do
        inst $file
    done
}

install_pam() {
    (
    if [[ "$LOOKS_LIKE_DEBIAN" ]] && type -p dpkg-architecture &>/dev/null; then
        find "/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security" -xtype f
    else
        find /lib*/security -xtype f
    fi
    find /etc/pam.d /etc/security -xtype f
    ) | while read file; do
        inst $file
    done

    # pam_unix depends on unix_chkpwd.
    # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
    dracut_install -o unix_chkpwd

    [[ "$LOOKS_LIKE_DEBIAN" ]] &&
        cp /etc/pam.d/systemd-user $initdir/etc/pam.d/

    # set empty root password for easy debugging
    sed -i 's/^root:x:/root::/' $initdir/etc/passwd
}

install_keymaps() {
    # The first three paths may be deprecated.
    # It seems now the last two paths are used by many distributions.
    for i in \
        /usr/lib/kbd/keymaps/include/* \
        /usr/lib/kbd/keymaps/i386/include/* \
        /usr/lib/kbd/keymaps/i386/qwerty/us.* \
        /usr/lib/kbd/keymaps/legacy/include/* \
        /usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
            [[ -f $i ]] || continue
            inst $i
    done

    # When it takes any argument, then install more keymaps.
    if [[ -n $1 ]]; then
        for i in \
        /usr/lib/kbd/keymaps/i386/*/* \
        /usr/lib/kbd/keymaps/legacy/i386/*/*; do
            [[ -f $i ]] || continue
            inst $i
        done
    fi
}

install_zoneinfo() {
    for i in /usr/share/zoneinfo/{,*/,*/*/}*; do
        [[ -f $i ]] || continue
        inst $i
    done
}

install_fonts() {
    for i in \
        /usr/lib/kbd/consolefonts/eurlatgr* \
        /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
            [[ -f $i ]] || continue
            inst $i
    done
}

install_terminfo() {
    for _terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
        [ -f ${_terminfodir}/l/linux ] && break
    done
    dracut_install -o ${_terminfodir}/l/linux
}

setup_testsuite() {
    cp $TEST_BASE_DIR/testsuite.target $initdir/etc/systemd/system/
    cp $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/

    mkdir -p $initdir/etc/systemd/system/testsuite.target.wants
    ln -fs $TEST_BASE_DIR/testsuite.service $initdir/etc/systemd/system/testsuite.target.wants/testsuite.service
    # Don't shutdown the machine after running the test when INTERACTIVE_DEBUG is set
    [[ -z $INTERACTIVE_DEBUG ]] && ln -fs $TEST_BASE_DIR/end.service $initdir/etc/systemd/system/testsuite.target.wants/end.service

    # make the testsuite the default target
    ln -fs testsuite.target $initdir/etc/systemd/system/default.target
}

setup_nspawn_root() {
    rm -fr $TESTDIR/nspawn-root
    ddebug "cp -ar $initdir $TESTDIR/nspawn-root"
    cp -ar $initdir $TESTDIR/nspawn-root
    # we don't mount in the nspawn root
    rm -f $TESTDIR/nspawn-root/etc/fstab
    if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
        cp -ar $TESTDIR/nspawn-root $TESTDIR/unprivileged-nspawn-root
    fi
}

setup_basic_dirs() {
    mkdir -p $initdir/run
    mkdir -p $initdir/etc/systemd/system
    mkdir -p $initdir/var/log/journal

    for d in usr/bin usr/sbin bin etc lib "$libdir" sbin tmp usr var var/log dev proc sys sysroot root run run/lock run/initramfs; do
        if [ -L "/$d" ]; then
            inst_symlink "/$d"
        else
            inst_dir "/$d"
        fi
    done

    ln -sfn /run "$initdir/var/run"
    ln -sfn /run/lock "$initdir/var/lock"
}

inst_libs() {
    local _bin=$1
    local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
    local _file _line

    LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
        [[ $_line = 'not a dynamic executable' ]] && break

        if [[ $_line =~ $_so_regex ]]; then
            _file=${BASH_REMATCH[1]}
            [[ -e ${initdir}/$_file ]] && continue
            inst_library "$_file"
            continue
        fi

        if [[ $_line =~ not\ found ]]; then
            dfatal "Missing a shared library required by $_bin."
            dfatal "Run \"ldd $_bin\" to find out what it is."
            dfatal "$_line"
            dfatal "dracut cannot create an initrd."
            exit 1
        fi
    done
}

import_testdir() {
    [[ -e $STATEFILE ]] && . $STATEFILE
    if [[ ! -d "$TESTDIR" ]]; then
        if [[ -z "$TESTDIR" ]]; then
            TESTDIR=$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)
        else
            mkdir -p "$TESTDIR"
        fi

        echo "TESTDIR=\"$TESTDIR\"" > $STATEFILE
        export TESTDIR
    fi
}

import_initdir() {
    initdir=$TESTDIR/root
    export initdir
}

## @brief Converts numeric logging level to the first letter of level name.
#
# @param lvl Numeric logging level in range from 1 to 6.
# @retval 1 if @a lvl is out of range.
# @retval 0 if @a lvl is correct.
# @result Echoes first letter of level name.
_lvl2char() {
    case "$1" in
        1) echo F;;
        2) echo E;;
        3) echo W;;
        4) echo I;;
        5) echo D;;
        6) echo T;;
        *) return 1;;
    esac
}

## @brief Internal helper function for _do_dlog()
#
# @param lvl Numeric logging level.
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
#
# @note This function is not supposed to be called manually. Please use
# dtrace(), ddebug(), or others instead which wrap this one.
#
# This function calls _do_dlog() either with parameter msg, or if
# none is given, it will read standard input and will use every line as
# a message.
#
# This enables:
# dwarn "This is a warning"
# echo "This is a warning" | dwarn
LOG_LEVEL=${LOG_LEVEL:-4}

dlog() {
    [ -z "$LOG_LEVEL" ] && return 0
    [ $1 -le $LOG_LEVEL ] || return 0
    local lvl="$1"; shift
    local lvlc=$(_lvl2char "$lvl") || return 0

    if [ $# -ge 1 ]; then
        echo "$lvlc: $*"
    else
        while read line; do
            echo "$lvlc: " "$line"
        done
    fi
}

## @brief Logs message at TRACE level (6)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dtrace() {
    set +x
    dlog 6 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at DEBUG level (5)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
ddebug() {
#    set +x
    dlog 5 "$@"
#    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at INFO level (4)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dinfo() {
    set +x
    dlog 4 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at WARN level (3)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dwarn() {
    set +x
    dlog 3 "$@"
    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at ERROR level (2)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
derror() {
#    set +x
    dlog 2 "$@"
#    [ -n "$debug" ] && set -x || :
}

## @brief Logs message at FATAL level (1)
#
# @param msg Message.
# @retval 0 It's always returned, even if logging failed.
dfatal() {
    set +x
    dlog 1 "$@"
    [ -n "$debug" ] && set -x || :
}


# Generic substring function.  If $2 is in $1, return 0.
strstr() { [ "${1#*$2*}" != "$1" ]; }

# normalize_path <path>
# Prints the normalized path, where it removes any duplicated
# and trailing slashes.
# Example:
# $ normalize_path ///test/test//
# /test/test
normalize_path() {
    shopt -q -s extglob
    set -- "${1//+(\/)//}"
    shopt -q -u extglob
    echo "${1%/}"
}

# convert_abs_rel <from> <to>
# Prints the relative path, when creating a symlink to <to> from <from>.
# Example:
# $ convert_abs_rel /usr/bin/test /bin/test-2
# ../../bin/test-2
# $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
convert_abs_rel() {
    local __current __absolute __abssize __cursize __newpath
    local -i __i __level

    set -- "$(normalize_path "$1")" "$(normalize_path "$2")"

    # corner case #1 - self looping link
    [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }

    # corner case #2 - own dir link
    [[ "${1%/*}" == "$2" ]] && { echo "."; return; }

    IFS="/" __current=($1)
    IFS="/" __absolute=($2)

    __abssize=${#__absolute[@]}
    __cursize=${#__current[@]}

    while [[ ${__absolute[__level]} == ${__current[__level]} ]]
    do
        (( __level++ ))
        if (( __level > __abssize || __level > __cursize ))
        then
            break
        fi
    done

    for ((__i = __level; __i < __cursize-1; __i++))
    do
        if ((__i > __level))
        then
            __newpath=$__newpath"/"
        fi
        __newpath=$__newpath".."
    done

    for ((__i = __level; __i < __abssize; __i++))
    do
        if [[ -n $__newpath ]]
        then
            __newpath=$__newpath"/"
        fi
        __newpath=$__newpath${__absolute[__i]}
    done

    echo "$__newpath"
}


# Install a directory, keeping symlinks as on the original system.
# Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
# will create ${initdir}/lib64, ${initdir}/lib64/file,
# and a symlink ${initdir}/lib -> lib64.
inst_dir() {
    [[ -e ${initdir}/"$1" ]] && return 0  # already there

    local _dir="$1" _part="${1%/*}" _file
    while [[ "$_part" != "${_part%/*}" ]] && ! [[ -e "${initdir}/${_part}" ]]; do
        _dir="$_part $_dir"
        _part=${_part%/*}
    done

    # iterate over parent directories
    for _file in $_dir; do
        [[ -e "${initdir}/$_file" ]] && continue
        if [[ -L $_file ]]; then
            inst_symlink "$_file"
        else
            # create directory
            mkdir -m 0755 -p "${initdir}/$_file" || return 1
            [[ -e "$_file" ]] && chmod --reference="$_file" "${initdir}/$_file"
            chmod u+w "${initdir}/$_file"
        fi
    done
}

# $1 = file to copy to ramdisk
# $2 (optional) Name for the file on the ramdisk
# Location of the image dir is assumed to be $initdir
# We never overwrite the target if it exists.
inst_simple() {
    [[ -f "$1" ]] || return 1
    strstr "$1" "/" || return 1

    local _src=$1 target="${2:-$1}"
    if ! [[ -d ${initdir}/$target ]]; then
        [[ -e ${initdir}/$target ]] && return 0
        [[ -L ${initdir}/$target ]] && return 0
        [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
    fi
    # install checksum files also
    if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
        inst "${_src%/*}/.${_src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
    fi
    ddebug "Installing $_src"
    cp --sparse=always -pfL "$_src" "${initdir}/$target"
}

# find symlinks linked to given library file
# $1 = library file
# Function searches for symlinks by stripping version numbers appended to
# library filename, checks if it points to the same target and finally
# prints the list of symlinks to stdout.
#
# Example:
# rev_lib_symlinks libfoo.so.8.1
# output: libfoo.so.8 libfoo.so
# (Only if libfoo.so.8 and libfoo.so exists on host system.)
rev_lib_symlinks() {
    [[ ! $1 ]] && return 0

    local fn="$1" orig="$(readlink -f "$1")" links=''

    [[ ${fn} =~ .*\.so\..* ]] || return 1

    until [[ ${fn##*.} == so ]]; do
        fn="${fn%.*}"
        [[ -L ${fn} && $(readlink -f "${fn}") == ${orig} ]] && links+=" ${fn}"
    done

    echo "${links}"
}

# Same as above, but specialized to handle dynamic libraries.
# It handles making symlinks according to how the original library
# is referenced.
inst_library() {
    local _src="$1" _dest=${2:-$1} _lib _reallib _symlink
    strstr "$1" "/" || return 1
    [[ -e $initdir/$_dest ]] && return 0
    if [[ -L $_src ]]; then
        # install checksum files also
        if [[ -e "${_src%/*}/.${_src##*/}.hmac" ]]; then
            inst "${_src%/*}/.${_src##*/}.hmac" "${_dest%/*}/.${_dest##*/}.hmac"
        fi
        _reallib=$(readlink -f "$_src")
        inst_simple "$_reallib" "$_reallib"
        inst_dir "${_dest%/*}"
        [[ -d "${_dest%/*}" ]] && _dest=$(readlink -f "${_dest%/*}")/${_dest##*/}
        ln -sfn $(convert_abs_rel "${_dest}" "${_reallib}") "${initdir}/${_dest}"
    else
        inst_simple "$_src" "$_dest"
    fi

    # Create additional symlinks.  See rev_symlinks description.
    for _symlink in $(rev_lib_symlinks $_src) $(rev_lib_symlinks $_reallib); do
        [[ -e $initdir/$_symlink ]] || {
            ddebug "Creating extra symlink: $_symlink"
            inst_symlink $_symlink
        }
    done
}

# find a binary.  If we were not passed the full path directly,
# search in the usual places to find the binary.
find_binary() {
    if [[ -z ${1##/*} ]]; then
        if [[ -x $1 ]] || { strstr "$1" ".so" && ldd $1 &>/dev/null; };  then
            echo $1
            return 0
        fi
    fi

    type -P $1
}

# Same as above, but specialized to install binary executables.
# Install binary executable, and all shared library dependencies, if any.
inst_binary() {
    local _bin _target
    _bin=$(find_binary "$1") || return 1
    _target=${2:-$_bin}
    [[ -e $initdir/$_target ]] && return 0
    [[ -L $_bin ]] && inst_symlink $_bin $_target && return 0
    local _file _line
    local _so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
    # I love bash!
    LC_ALL=C ldd "$_bin" 2>/dev/null | while read _line; do
        [[ $_line = 'not a dynamic executable' ]] && break

        if [[ $_line =~ $_so_regex ]]; then
            _file=${BASH_REMATCH[1]}
            [[ -e ${initdir}/$_file ]] && continue
            inst_library "$_file"
            continue
        fi

        if [[ $_line =~ not\ found ]]; then
            dfatal "Missing a shared library required by $_bin."
            dfatal "Run \"ldd $_bin\" to find out what it is."
            dfatal "$_line"
            dfatal "dracut cannot create an initrd."
            exit 1
        fi
    done
    inst_simple "$_bin" "$_target"
}

# same as above, except for shell scripts.
# If your shell script does not start with shebang, it is not a shell script.
inst_script() {
    local _bin
    _bin=$(find_binary "$1") || return 1
    shift
    local _line _shebang_regex
    read -r -n 80 _line <"$_bin"
    # If debug is set, clean unprintable chars to prevent messing up the term
    [[ $debug ]] && _line=$(echo -n "$_line" | tr -c -d '[:print:][:space:]')
    _shebang_regex='(#! *)(/[^ ]+).*'
    [[ $_line =~ $_shebang_regex ]] || return 1
    inst "${BASH_REMATCH[2]}" && inst_simple "$_bin" "$@"
}

# same as above, but specialized for symlinks
inst_symlink() {
    local _src=$1 _target=${2:-$1} _realsrc
    strstr "$1" "/" || return 1
    [[ -L $1 ]] || return 1
    [[ -L $initdir/$_target ]] && return 0
    _realsrc=$(readlink -f "$_src")
    if ! [[ -e $initdir/$_realsrc ]]; then
        if [[ -d $_realsrc ]]; then
            inst_dir "$_realsrc"
        else
            inst "$_realsrc"
        fi
    fi
    [[ ! -e $initdir/${_target%/*} ]] && inst_dir "${_target%/*}"
    [[ -d ${_target%/*} ]] && _target=$(readlink -f ${_target%/*})/${_target##*/}
    ln -sfn $(convert_abs_rel "${_target}" "${_realsrc}") "$initdir/$_target"
}

# attempt to install any programs specified in a udev rule
inst_rule_programs() {
    local _prog _bin

    if grep -qE 'PROGRAM==?"[^ "]+' "$1"; then
        for _prog in $(grep -E 'PROGRAM==?"[^ "]+' "$1" | sed -r 's/.*PROGRAM==?"([^ "]+).*/\1/'); do
            if [ -x /lib/udev/$_prog ]; then
                _bin=/lib/udev/$_prog
            else
                _bin=$(find_binary "$_prog") || {
                    dinfo "Skipping program $_prog using in udev rule $(basename $1) as it cannot be found"
                    continue;
                }
            fi

            #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
            dracut_install "$_bin"
        done
    fi
}

# udev rules always get installed in the same place, so
# create a function to install them to make life simpler.
inst_rules() {
    local _target=/etc/udev/rules.d _rule _found

    inst_dir "/lib/udev/rules.d"
    inst_dir "$_target"
    for _rule in "$@"; do
        if [ "${rule#/}" = "$rule" ]; then
            for r in /lib/udev/rules.d /etc/udev/rules.d; do
                if [[ -f $r/$_rule ]]; then
                    _found="$r/$_rule"
                    inst_simple "$_found"
                    inst_rule_programs "$_found"
                fi
            done
        fi
        for r in '' ./ $dracutbasedir/rules.d/; do
            if [[ -f ${r}$_rule ]]; then
                _found="${r}$_rule"
                inst_simple "$_found" "$_target/${_found##*/}"
                inst_rule_programs "$_found"
            fi
        done
        [[ $_found ]] || dinfo "Skipping udev rule: $_rule"
        _found=
    done
}

# general purpose installation function
# Same args as above.
inst() {
    local _x

    case $# in
        1) ;;
        2) [[ ! $initdir && -d $2 ]] && export initdir=$2
            [[ $initdir = $2 ]] && set $1;;
        3) [[ -z $initdir ]] && export initdir=$2
            set $1 $3;;
        *) dfatal "inst only takes 1 or 2 or 3 arguments"
            exit 1;;
    esac
    for _x in inst_symlink inst_script inst_binary inst_simple; do
        $_x "$@" && return 0
    done
    return 1
}

# install any of listed files
#
# If first argument is '-d' and second some destination path, first accessible
# source is installed into this path, otherwise it will installed in the same
# path as source.  If none of listed files was installed, function return 1.
# On first successful installation it returns with 0 status.
#
# Example:
#
# inst_any -d /bin/foo /bin/bar /bin/baz
#
# Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
# initramfs.
inst_any() {
    local to f

    [[ $1 = '-d' ]] && to="$2" && shift 2

    for f in "$@"; do
        if [[ -e $f ]]; then
            [[ $to ]] && inst "$f" "$to" && return 0
            inst "$f" && return 0
        fi
    done

    return 1
}

# dracut_install [-o ] <file> [<file> ... ]
# Install <file> to the initramfs image
# -o optionally install the <file> and don't fail, if it is not there
dracut_install() {
    local _optional=no
    if [[ $1 = '-o' ]]; then
        _optional=yes
        shift
    fi
    while (($# > 0)); do
        if ! inst "$1" ; then
            if [[ $_optional = yes ]]; then
                dinfo "Skipping program $1 as it cannot be found and is" \
                    "flagged to be optional"
            else
                dfatal "Failed to install $1"
                exit 1
            fi
        fi
        shift
    done
}

# Install a single kernel module along with any firmware it may require.
# $1 = full path to kernel module to install
install_kmod_with_fw() {
    # no need to go further if the module is already installed

    [[ -e "${initdir}/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" ]] \
        && return 0

    [[ -e "$initdir/.kernelmodseen/${1##*/}" ]] && return 0

    if [[ $omit_drivers ]]; then
        local _kmod=${1##*/}
        _kmod=${_kmod%.ko}
        _kmod=${_kmod/-/_}
        if [[ "$_kmod" =~ $omit_drivers ]]; then
            dinfo "Omitting driver $_kmod"
            return 1
        fi
        if [[ "${1##*/lib/modules/$KERNEL_VER/}" =~ $omit_drivers ]]; then
            dinfo "Omitting driver $_kmod"
            return 1
        fi
    fi

    [ -d "$initdir/.kernelmodseen" ] && \
        > "$initdir/.kernelmodseen/${1##*/}"

    inst_simple "$1" "/lib/modules/$KERNEL_VER/${1##*/lib/modules/$KERNEL_VER/}" \
        || return $?

    local _modname=${1##*/} _fwdir _found _fw
    _modname=${_modname%.ko*}
    for _fw in $(modinfo -k $KERNEL_VER -F firmware $1 2>/dev/null); do
        _found=''
        for _fwdir in $fw_dir; do
            if [[ -d $_fwdir && -f $_fwdir/$_fw ]]; then
                inst_simple "$_fwdir/$_fw" "/lib/firmware/$_fw"
                _found=yes
            fi
        done
        if [[ $_found != yes ]]; then
            if ! grep -qe "\<${_modname//-/_}\>" /proc/modules; then
                dinfo "Possible missing firmware \"${_fw}\" for kernel module" \
                    "\"${_modname}.ko\""
            else
                dwarn "Possible missing firmware \"${_fw}\" for kernel module" \
                    "\"${_modname}.ko\""
            fi
        fi
    done
    return 0
}

# Do something with all the dependencies of a kernel module.
# Note that kernel modules depend on themselves using the technique we use
# $1 = function to call for each dependency we find
#      It will be passed the full path to the found kernel module
# $2 = module to get dependencies for
# rest of args = arguments to modprobe
# _fderr specifies FD passed from surrounding scope
for_each_kmod_dep() {
    local _func=$1 _kmod=$2 _cmd _modpath _options _found=0
    shift 2
    modprobe "$@" --ignore-install --show-depends $_kmod 2>&${_fderr} | (
        while read _cmd _modpath _options; do
            [[ $_cmd = insmod ]] || continue
            $_func ${_modpath} || exit $?
            _found=1
        done
        [[ $_found -eq 0 ]] && exit 1
        exit 0
    )
}

# filter kernel modules to install certain modules that meet specific
# requirements.
# $1 = search only in subdirectory of /kernel/$1
# $2 = function to call with module name to filter.
#      This function will be passed the full path to the module to test.
# The behavior of this function can vary depending on whether $hostonly is set.
# If it is, we will only look at modules that are already in memory.
# If it is not, we will look at all kernel modules
# This function returns the full filenames of modules that match $1
filter_kernel_modules_by_path () (
    local _modname _filtercmd
    if ! [[ $hostonly ]]; then
        _filtercmd='find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra"'
        _filtercmd+=' "$KERNEL_MODS/weak-updates" -name "*.ko" -o -name "*.ko.gz"'
        _filtercmd+=' -o -name "*.ko.xz"'
        _filtercmd+=' 2>/dev/null'
    else
        _filtercmd='cut -d " " -f 1 </proc/modules|xargs modinfo -F filename '
        _filtercmd+='-k $KERNEL_VER 2>/dev/null'
    fi
    for _modname in $(eval $_filtercmd); do
        case $_modname in
            *.ko) "$2" "$_modname" && echo "$_modname";;
            *.ko.gz) gzip -dc "$_modname" > $initdir/$$.ko
                $2 $initdir/$$.ko && echo "$_modname"
                rm -f $initdir/$$.ko
                ;;
            *.ko.xz) xz -dc "$_modname" > $initdir/$$.ko
                $2 $initdir/$$.ko && echo "$_modname"
                rm -f $initdir/$$.ko
                ;;
        esac
    done
)
find_kernel_modules_by_path () (
    if ! [[ $hostonly ]]; then
        find "$KERNEL_MODS/kernel/$1" "$KERNEL_MODS/extra" "$KERNEL_MODS/weak-updates" \
          -name "*.ko" -o -name "*.ko.gz" -o -name "*.ko.xz" 2>/dev/null
    else
        cut -d " " -f 1 </proc/modules \
        | xargs modinfo -F filename -k $KERNEL_VER 2>/dev/null
    fi
)

filter_kernel_modules () {
    filter_kernel_modules_by_path  drivers  "$1"
}

find_kernel_modules () {
    find_kernel_modules_by_path  drivers
}

# instmods [-c] <kernel module> [<kernel module> ... ]
# instmods [-c] <kernel subsystem>
# install kernel modules along with all their dependencies.
# <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
instmods() {
    [[ $no_kernel = yes ]] && return
    # called [sub]functions inherit _fderr
    local _fderr=9
    local _check=no
    if [[ $1 = '-c' ]]; then
        _check=yes
        shift
    fi

    function inst1mod() {
        local _ret=0 _mod="$1"
        case $_mod in
            =*)
                if [ -f $KERNEL_MODS/modules.${_mod#=} ]; then
                    ( [[ "$_mpargs" ]] && echo $_mpargs
                      cat "${KERNEL_MODS}/modules.${_mod#=}" ) \
                    | instmods
                else
                    ( [[ "$_mpargs" ]] && echo $_mpargs
                      find "$KERNEL_MODS" -path "*/${_mod#=}/*" -printf '%f\n' ) \
                    | instmods
                fi
                ;;
            --*) _mpargs+=" $_mod" ;;
            i2o_scsi) return ;; # Do not load this diagnostic-only module
            *)
                _mod=${_mod##*/}
                # if we are already installed, skip this module and go on
                # to the next one.
                [[ -f "$initdir/.kernelmodseen/${_mod%.ko}.ko" ]] && return

                if [[ $omit_drivers ]] && [[ "$1" =~ $omit_drivers ]]; then
                    dinfo "Omitting driver ${_mod##$KERNEL_MODS}"
                    return
                fi
                # If we are building a host-specific initramfs and this
                # module is not already loaded, move on to the next one.
                [[ $hostonly ]] && ! grep -qe "\<${_mod//-/_}\>" /proc/modules \
                    && ! echo $add_drivers | grep -qe "\<${_mod}\>" \
                    && return

                # We use '-d' option in modprobe only if modules prefix path
                # differs from default '/'.  This allows us to use Dracut with
                # old version of modprobe which doesn't have '-d' option.
                local _moddirname=${KERNEL_MODS%%/lib/modules/*}
                [[ -n ${_moddirname} ]] && _moddirname="-d ${_moddirname}/"

                # ok, load the module, all its dependencies, and any firmware
                # it may require
                for_each_kmod_dep install_kmod_with_fw $_mod \
                    --set-version $KERNEL_VER ${_moddirname} $_mpargs
                ((_ret+=$?))
                ;;
        esac
        return $_ret
    }

    function instmods_1() {
        local _mod _mpargs
        if (($# == 0)); then  # filenames from stdin
            while read _mod; do
                inst1mod "${_mod%.ko*}" || {
                    if [ "$_check" = "yes" ]; then
                        dfatal "Failed to install $_mod"
                        return 1
                    fi
                }
            done
        fi
        while (($# > 0)); do  # filenames as arguments
            inst1mod ${1%.ko*} || {
                if [ "$_check" = "yes" ]; then
                    dfatal "Failed to install $1"
                    return 1
                fi
            }
            shift
        done
        return 0
    }

    local _ret _filter_not_found='FATAL: Module .* not found.'
    set -o pipefail
    # Capture all stderr from modprobe to _fderr. We could use {var}>...
    # redirections, but that would make dracut require bash4 at least.
    eval "( instmods_1 \"\$@\" ) ${_fderr}>&1" \
    | while read line; do [[ "$line" =~ $_filter_not_found ]] && echo $line || echo $line >&2 ;done | derror
    _ret=$?
    set +o pipefail
    return $_ret
}

# inst_libdir_file [-n <pattern>] <file> [<file>...]
# Install a <file> located on a lib directory to the initramfs image
# -n <pattern> install non-matching files
inst_libdir_file() {
    if [[ "$1" == "-n" ]]; then
        local _pattern=$1
        shift 2
        for _dir in $libdirs; do
            for _i in "$@"; do
                for _f in "$_dir"/$_i; do
                    [[ "$_i" =~ $_pattern ]] || continue
                    [[ -e "$_i" ]] && dracut_install "$_i"
                done
            done
        done
    else
        for _dir in $libdirs; do
            for _i in "$@"; do
                for _f in "$_dir"/$_i; do
                    [[ -e "$_f" ]] && dracut_install "$_f"
                done
            done
        done
    fi
}

setup_suse() {
    ln -fs ../usr/bin/systemctl $initdir/bin/
    ln -fs ../usr/lib/systemd $initdir/lib/
    inst_simple "/usr/lib/systemd/system/haveged.service"
}

# can be overridden in specific test
test_cleanup() {
    umount $TESTDIR/root 2>/dev/null || true
    [[ $LOOPDEV ]] && losetup -d $LOOPDEV || true
    return 0
}

test_run() {
    if [ -z "$TEST_NO_QEMU" ]; then
        if run_qemu; then
            check_result_qemu || return 1
        else
            dwarn "can't run QEMU, skipping"
        fi
    fi
    if [ -z "$TEST_NO_NSPAWN" ]; then
        if run_nspawn "nspawn-root"; then
            check_result_nspawn "nspawn-root" || return 1
        else
            dwarn "can't run systemd-nspawn, skipping"
        fi

        if [[ "$RUN_IN_UNPRIVILEGED_CONTAINER" = "yes" ]]; then
            if NSPAWN_ARGUMENTS="-U --private-network $NSPAWN_ARGUMENTS" run_nspawn "unprivileged-nspawn-root"; then
                check_result_nspawn "unprivileged-nspawn-root" || return 1
            else
                dwarn "can't run systemd-nspawn, skipping"
            fi
        fi
    fi
    return 0
}

do_test() {
    if [[ $UID != "0" ]]; then
        echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
        exit 0
    fi

    # Detect lib paths
    [[ $libdir ]] || for libdir in /lib64 /lib; do
        [[ -d $libdir ]] && libdirs+=" $libdir" && break
    done

    [[ $usrlibdir ]] || for usrlibdir in /usr/lib64 /usr/lib; do
        [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
    done

    mkdir -p "$STATEDIR"

    import_testdir
    import_initdir

    while (($# > 0)); do
        case $1 in
            --run)
                echo "TEST RUN: $TEST_DESCRIPTION"
                test_run
                ret=$?
                if (( $ret == 0 )); then
                    echo "TEST RUN: $TEST_DESCRIPTION [OK]"
                else
                    echo "TEST RUN: $TEST_DESCRIPTION [FAILED]"
                fi
                exit $ret;;
            --setup)
                echo "TEST SETUP: $TEST_DESCRIPTION"
                test_setup
                ;;
            --clean)
                echo "TEST CLEANUP: $TEST_DESCRIPTION"
                test_cleanup
                rm -fr "$TESTDIR"
                rm -f "$STATEFILE"
                ;;
            --all)
                ret=0
                echo -n "TEST: $TEST_DESCRIPTION ";
                (
                    test_setup && test_run
                    ret=$?
                    test_cleanup
                    rm -fr "$TESTDIR"
                    rm -f "$STATEFILE"
                    exit $ret
                ) </dev/null >"$TESTLOG" 2>&1 || ret=$?
                if [ $ret -eq 0 ]; then
                    rm "$TESTLOG"
                    echo "[OK]"
                else
                    echo "[FAILED]"
                    echo "see $TESTLOG"
                fi
                exit $ret;;
            *) break ;;
        esac
        shift
    done
}