test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  18 export PATH
  19
  20 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  21 # shellcheck source=/dev/null
  22 source "$os_release"
  23 [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=""
  24 [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=""
  25 [[ " $ID_LIKE " = *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=""
  26 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  27 QEMU_TIMEOUT="${QEMU_TIMEOUT:-infinity}"
  28 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-infinity}"
  29 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  30 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  31 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  32 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  33 QEMU_MEM="${QEMU_MEM:-512M}"
  34 # Note that defining a different IMAGE_NAME in a test setup script will only result
  35 # in default.img being copied and renamed. It can then be extended by defining
  36 # a test_append_files() function. The $1 parameter will be the root directory.
  37 # To force creating a new image from scratch (eg: to encrypt it), also define
  38 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  39 IMAGE_NAME=${IMAGE_NAME:-default}
  40 STRIP_BINARIES="${STRIP_BINARIES:-yes}"
  41 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  42 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  43 LOOPDEV=
  44
  45 # Simple wrapper to unify boolean checks.
  46 # Note: this function needs to stay near the top of the file, so we can use it
  47 #       in code in the outermost scope.
  48 get_bool() {
  49     # Make the value lowercase to make the regex matching simpler
  50     local _bool="${1,,}"
  51
  52     # Consider empty value as "false"
  53     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  54         return 1
  55     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  56         return 0
  57     else
  58         echo >&2 "Value '$_bool' is not a valid boolean value"
  59         exit 1
  60     fi
  61 }
  62
  63 # Decide if we can (and want to) run QEMU with KVM acceleration.
  64 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  65 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  66 # running under KVM. If these conditions are met, enable KVM (and possibly
  67 # nested KVM), otherwise disable it.
  68 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && [[ "$(systemd-detect-virt -v)" != kvm ]]); then
  69     QEMU_KVM=yes
  70 else
  71     QEMU_KVM=no
  72 fi
  73
  74 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  75     echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
  76     ROOTLIBDIR=/usr/lib/systemd
  77 fi
  78
  79 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
  80 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
  81 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
  82 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
  83 TOOLS_DIR="$SOURCE_DIR/tools"
  84 # These variables are used by test scripts
  85 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR TOOLS_DIR
  86
  87 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
  88 if get_bool "${NO_BUILD:=}"; then
  89     BUILD_DIR="$SOURCE_DIR"
  90 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
  91     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
  92     exit 1
  93 fi
  94
  95 PATH_TO_INIT="$ROOTLIBDIR/systemd"
  96 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
  97 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote")}"
  98 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
  99 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 100 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 101
 102 TESTFILE="${BASH_SOURCE[1]}"
 103 if [ -z "$TESTFILE" ]; then
 104     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 105     exit 1
 106 fi
 107 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 108 STATEDIR="$BUILD_DIR/test/$TESTNAME"
 109 STATEFILE="$STATEDIR/.testdir"
 110 IMAGESTATEDIR="$STATEDIR/.."
 111 TESTLOG="$STATEDIR/test.log"
 112
 113 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 114     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 115     exit 1
 116 fi
 117 TESTID="${BASH_REMATCH[1]:?}"
 118
 119 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 120     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 121     exit 1
 122 fi
 123
 124 BASICTOOLS=(
 125     awk
 126     base64
 127     basename
 128     bash
 129     capsh
 130     cat
 131     chmod
 132     chown
 133     cmp
 134     cryptsetup
 135     cut
 136     date
 137     dd
 138     diff
 139     dirname
 140     dmsetup
 141     echo
 142     env
 143     false
 144     getconf
 145     getent
 146     getfacl
 147     grep
 148     gunzip
 149     gzip
 150     head
 151     ionice
 152     ip
 153     ldd
 154     ln
 155     loadkeys
 156     login
 157     losetup
 158     lz4cat
 159     mkfifo
 160     mktemp
 161     modprobe
 162     mount
 163     mountpoint
 164     mv
 165     nc
 166     nproc
 167     readlink
 168     rev
 169     rm
 170     rmdir
 171     sed
 172     seq
 173     setfattr
 174     setfont
 175     setsid
 176     sfdisk
 177     sh
 178     sleep
 179     stat
 180     su
 181     sulogin
 182     sysctl
 183     tail
 184     tar
 185     tee
 186     test
 187     timeout
 188     touch
 189     tr
 190     true
 191     truncate
 192     umount
 193     uname
 194     unshare
 195     wc
 196     xargs
 197     xzcat
 198 )
 199
 200 DEBUGTOOLS=(
 201     cp
 202     df
 203     dhclient
 204     dmesg
 205     du
 206     find
 207     free
 208     grep
 209     hostname
 210     id
 211     less
 212     ln
 213     ls
 214     mkdir
 215     ping
 216     ps
 217     route
 218     sort
 219     strace
 220     stty
 221     tty
 222     vi
 223 )
 224
 225 is_built_with_asan() {
 226     if ! type -P objdump >/dev/null; then
 227         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
 228         return 1
 229     fi
 230
 231     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 232     local _asan_calls
 233     _asan_calls="$(objdump -dC "$SYSTEMD_JOURNALD" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 234     if ((_asan_calls < 1000)); then
 235         return 1
 236     else
 237         return 0
 238     fi
 239 }
 240
 241 is_built_with_coverage() {
 242     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 243         return 1
 244     fi
 245
 246     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 247 }
 248
 249 IS_BUILT_WITH_ASAN=$(is_built_with_asan && echo yes || echo no)
 250 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 251
 252 if get_bool "$IS_BUILT_WITH_ASAN"; then
 253     STRIP_BINARIES=no
 254     SKIP_INITRD="${SKIP_INITRD:-yes}"
 255     PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
 256     QEMU_MEM="2048M"
 257     QEMU_SMP="${QEMU_SMP:-4}"
 258
 259     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 260     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 261         ASAN_COMPILER=gcc
 262         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 263     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 264         ASAN_COMPILER=clang
 265         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 266
 267         # As clang's ASan DSO is usually in a non-standard path, let's check if
 268         # the environment is set accordingly. If not, warn the user and exit.
 269         # We're not setting the LD_LIBRARY_PATH automagically here, because
 270         # user should encounter (and fix) the same issue when running the unit
 271         # tests (meson test)
 272         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 273             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 274             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 275             exit 1
 276         fi
 277     else
 278         echo >&2 "systemd is not linked against the ASan DSO"
 279         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 280         exit 1
 281     fi
 282
 283     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 284 fi
 285
 286 find_qemu_bin() {
 287     QEMU_BIN="${QEMU_BIN:-""}"
 288     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 289     if get_bool "$QEMU_KVM"; then
 290         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 291     fi
 292
 293     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 294     case $ARCH in
 295     x86_64)
 296         # QEMU's own build system calls it qemu-system-x86_64
 297         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 298         ;;
 299     i*86)
 300         # new i386 version of QEMU
 301         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 302
 303         # i386 version of QEMU
 304         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 305         ;;
 306     ppc64*)
 307         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 308         ;;
 309     esac
 310
 311     if [[ ! -e "$QEMU_BIN" ]]; then
 312         echo "Could not find a suitable QEMU binary" >&2
 313         return 1
 314     fi
 315 }
 316
 317 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 318 # returns 0 if newer or equal
 319 # returns 1 if older
 320 # returns 2 if failing
 321 qemu_min_version() {
 322     find_qemu_bin || return 2
 323
 324     # get version from binary
 325     local qemu_ver
 326     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 327
 328     # Check version string format
 329     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 330     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 331
 332     # compare as last command to return that value
 333     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 334 }
 335
 336 # Return 0 if QEMU did run (then you must check the result state/logs for actual
 337 # success), or 1 if QEMU is not available.
 338 run_qemu() {
 339     if [ -f /etc/machine-id ]; then
 340         read -r MACHINE_ID </etc/machine-id
 341         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 342             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 343         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 344             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 345     fi
 346
 347     local CONSOLE=ttyS0
 348
 349     rm -f "$initdir"/{testok,failed,skipped}
 350     # make sure the initdir is not mounted to avoid concurrent access
 351     cleanup_initdir
 352     umount_loopback
 353
 354     if [[ ! "$KERNEL_BIN" ]]; then
 355         if get_bool "$LOOKS_LIKE_ARCH"; then
 356             KERNEL_BIN=/boot/vmlinuz-linux
 357         else
 358             [ "$ARCH" ] || ARCH=$(uname -m)
 359             case $ARCH in
 360                 ppc64*)
 361                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 362                     # uses the "standard" vmlinuz- prefix
 363                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 364                     CONSOLE=hvc0
 365                     ;;
 366                 *)
 367                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 368                     ;;
 369             esac
 370         fi
 371     fi
 372
 373     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 374     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 375     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 376     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 377     if [[ ! "$INITRD" ]]; then
 378         if [[ -e "$default_fedora_initrd" ]]; then
 379             INITRD="$default_fedora_initrd"
 380         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 381             INITRD="$default_debian_initrd"
 382         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 383             INITRD="$default_arch_initrd"
 384         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 385             INITRD="$default_suse_initrd"
 386         fi
 387     fi
 388
 389     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 390     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 391     # is not installed or there's some other error when calling it, fall back
 392     # to the original value (QEMU_SMP=1).
 393     if [[ -z "${QEMU_SMP:=}" ]]; then
 394         if ! QEMU_SMP=$(nproc); then
 395             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 396             QEMU_SMP=1
 397         fi
 398     fi
 399
 400     find_qemu_bin || return 1
 401
 402     # Umount initdir to avoid concurrent access to the filesystem
 403     _umount_dir "$initdir"
 404
 405     local kernel_params=()
 406     local qemu_options=()
 407     local qemu_cmd=("$QEMU_BIN")
 408
 409     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 410         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 411     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 412         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 413     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 414         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 415     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 416         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 417         exit 1
 418     fi
 419
 420     if get_bool "$LOOKS_LIKE_SUSE"; then
 421         kernel_params+=("rd.hostonly=0")
 422     fi
 423
 424     kernel_params+=(
 425         "root=LABEL=systemd_boot"
 426         "rw"
 427         "raid=noautodetect"
 428         "rd.luks=0"
 429         "loglevel=2"
 430         "init=$PATH_TO_INIT"
 431         "console=$CONSOLE"
 432         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 433         "systemd.unit=testsuite.target"
 434         "systemd.wants=testsuite-$1.service"
 435     )
 436
 437     if ! get_bool "$INTERACTIVE_DEBUG"; then
 438         kernel_params+=("systemd.wants=end.service")
 439     fi
 440
 441     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 442     qemu_options+=(
 443         -smp "$QEMU_SMP"
 444         -net none
 445         -m "$QEMU_MEM"
 446         -nographic
 447         -kernel "$KERNEL_BIN"
 448         -drive "format=raw,cache=unsafe,file=$image"
 449     )
 450
 451     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 452         local user_qemu_options
 453         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 454         qemu_options+=("${user_qemu_options[@]}")
 455     fi
 456
 457     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 458         local user_kernel_append
 459         read -ra user_kernel_append <<< "$KERNEL_APPEND"
 460         kernel_params+=("${user_kernel_append[@]}")
 461     fi
 462
 463     if [[ "$INITRD" ]] && ! get_bool "$SKIP_INITRD"; then
 464         qemu_options+=(-initrd "$INITRD")
 465     fi
 466
 467     # Let's use KVM if possible
 468     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 469         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 470     fi
 471
 472     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 473         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 474     fi
 475
 476     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]}")
 477     rc=$?
 478     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 479         derror "Test timed out after ${QEMU_TIMEOUT}s"
 480         TIMED_OUT=1
 481     else
 482         [ "$rc" != 0 ] && derror "QEMU failed with exit code $rc"
 483     fi
 484     return 0
 485 }
 486
 487 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 488 # success), or 1 if nspawn is not available.
 489 run_nspawn() {
 490     [[ -d /run/systemd/system ]] || return 1
 491     rm -f "${initdir:?}"/{testok,failed,skipped}
 492
 493     local nspawn_cmd=()
 494     local nspawn_options=(
 495         "--register=no"
 496         "--kill-signal=SIGKILL"
 497         "--directory=${1:?}"
 498         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 499     )
 500     local kernel_params=(
 501         "$PATH_TO_INIT"
 502         "systemd.unit=testsuite.target"
 503         "systemd.wants=testsuite-$2.service"
 504     )
 505
 506     if ! get_bool "$INTERACTIVE_DEBUG"; then
 507         kernel_params+=("systemd.wants=end.service")
 508     fi
 509
 510     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 511         local user_nspawn_arguments
 512         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 513         nspawn_options+=("${user_nspawn_arguments[@]}")
 514     fi
 515
 516     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 517         local user_kernel_append
 518         read -ra user_kernel_append <<< "$KERNEL_APPEND"
 519         kernel_params+=("${user_kernel_append[@]}")
 520     fi
 521
 522     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 523         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 524         exit
 525     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 526         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 527     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 528         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 529     else
 530         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 531         exit 1
 532     fi
 533
 534     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 535         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 536     else
 537         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 538     fi
 539
 540     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}")
 541     rc=$?
 542     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 543         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 544         TIMED_OUT=1
 545     else
 546         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 547     fi
 548     return 0
 549 }
 550
 551 # Build two very minimal root images, with two units, one is the same and one is different across them
 552 install_verity_minimal() {
 553     dinfo "Set up a set of minimal images for verity verification"
 554     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 555         return
 556     fi
 557     if ! command -v mksquashfs >/dev/null 2>&1; then
 558         dfatal "mksquashfs not found"
 559         exit 1
 560     fi
 561     if ! command -v veritysetup >/dev/null 2>&1; then
 562         dfatal "veritysetup not found"
 563         exit 1
 564     fi
 565     # Local modifications of some global variables is intentional in this
 566     # subshell (SC2030)
 567     # shellcheck disable=SC2030
 568     (
 569         BASICTOOLS=(
 570             bash
 571             cat
 572             grep
 573             mount
 574             sleep
 575         )
 576         oldinitdir="$initdir"
 577         rm -rfv "$TESTDIR/minimal"
 578         export initdir="$TESTDIR/minimal"
 579         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 580         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 581         setup_basic_dirs
 582         install_basic_tools
 583         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 584         # issue, thus falsely triggering SC2030 in this case
 585         # See: koalaman/shellcheck#1409
 586         if [[ -v ASAN_RT_PATH ]]; then
 587             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 588             # into the verity images to get rid of the annoying errors about
 589             # missing $LD_PRELOAD libraries.
 590             inst_libs "$ASAN_RT_PATH"
 591             inst_library "$ASAN_RT_PATH"
 592             # Create a dummy LSan suppression file otherwise gcc's ASan
 593             # complains as it doesn't exist in the minimal image
 594             # (i.e. when running TEST-29 or TEST-50 under sanitizers)
 595             touch "$initdir/systemd-lsan.supp"
 596         fi
 597         cp "$os_release" "$initdir/usr/lib/os-release"
 598         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 599         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 600         touch "$initdir/opt/some_file"
 601         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 602         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 603         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 604 [Service]
 605 ExecStartPre=cat /usr/lib/os-release
 606 ExecStart=sleep 120
 607 EOF
 608         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 609
 610         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 611         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 612             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 613
 614         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 615         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 616         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 617
 618         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 619         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 620             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 621
 622         # Rolling distros like Arch do not set VERSION_ID
 623         local version_id=""
 624         if grep -q "^VERSION_ID=" "$os_release"; then
 625             version_id="$(grep "^VERSION_ID=" "$os_release")"
 626         fi
 627
 628         export initdir="$TESTDIR/app0"
 629         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 630         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 631         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 632         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 633 [Service]
 634 Type=oneshot
 635 RemainAfterExit=yes
 636 ExecStart=/opt/script0.sh
 637 TemporaryFileSystem=/var/lib
 638 StateDirectory=app0
 639 RuntimeDirectory=app0
 640 EOF
 641         cat >"$initdir/opt/script0.sh" <<EOF
 642 #!/bin/bash
 643 set -e
 644 test -e /usr/lib/os-release
 645 echo bar > \${STATE_DIRECTORY}/foo
 646 cat /usr/lib/extension-release.d/extension-release.app0
 647 EOF
 648         chmod +x "$initdir/opt/script0.sh"
 649         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 650         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 651
 652         export initdir="$TESTDIR/app1"
 653         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 654         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 655         ( echo "${version_id}"
 656           echo "SYSEXT_SCOPE=portable"
 657           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 658         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 659         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 660 [Service]
 661 Type=oneshot
 662 RemainAfterExit=yes
 663 ExecStart=/opt/script1.sh
 664 StateDirectory=app1
 665 RuntimeDirectory=app1
 666 EOF
 667         cat >"$initdir/opt/script1.sh" <<EOF
 668 #!/bin/bash
 669 set -e
 670 test -e /usr/lib/os-release
 671 echo baz > \${STATE_DIRECTORY}/foo
 672 cat /usr/lib/extension-release.d/extension-release.app2
 673 EOF
 674         chmod +x "$initdir/opt/script1.sh"
 675         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 676         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 677     )
 678 }
 679
 680 setup_basic_environment() {
 681     # create the basic filesystem layout
 682     setup_basic_dirs
 683
 684     install_systemd
 685     install_missing_libraries
 686     install_config_files
 687     install_zoneinfo
 688     create_rc_local
 689     install_basic_tools
 690     install_libnss
 691     install_pam
 692     install_dbus
 693     install_fonts
 694     install_keymaps
 695     install_terminfo
 696     install_execs
 697     install_fs_tools
 698     install_modules
 699     install_plymouth
 700     install_haveged
 701     install_debug_tools
 702     install_ld_so_conf
 703     install_testuser
 704     has_user_dbus_socket && install_user_dbus
 705     setup_selinux
 706     strip_binaries
 707     instmods veth
 708     install_depmod_files
 709     generate_module_dependencies
 710     if get_bool "$IS_BUILT_WITH_ASAN"; then
 711         create_asan_wrapper
 712     fi
 713     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 714         install_verity_minimal
 715     fi
 716 }
 717
 718 setup_selinux() {
 719     dinfo "Setup SELinux"
 720     # don't forget KERNEL_APPEND='... selinux=1 ...'
 721     if ! get_bool "$SETUP_SELINUX"; then
 722         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 723         return 0
 724     fi
 725
 726     local conf_dir=/etc/selinux
 727     local fixfiles_tools=(bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles)
 728
 729     # Make sure the following statement can't expand to "/" to prevent
 730     # a potential where-are-my-backups situation
 731     rm -rf "${initdir:?}/$conf_dir"
 732     if ! cp -ar "$conf_dir" "$initdir/$conf_dir"; then
 733         dfatal "Failed to copy $conf_dir"
 734         exit 1
 735     fi
 736
 737     touch "$initdir/.autorelabel"
 738     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 739     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 740
 741     image_install "${fixfiles_tools[@]}"
 742     image_install fixfiles
 743     image_install sestatus
 744 }
 745
 746 install_valgrind() {
 747     if ! type -p valgrind; then
 748         dfatal "Failed to install valgrind"
 749         exit 1
 750     fi
 751
 752     local valgrind_bins valgrind_libs valgrind_dbg_and_supp
 753
 754     valgrind_bins="$(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')"
 755     image_install "$valgrind_bins"
 756
 757     valgrind_libs="$(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')"
 758     image_install "$valgrind_libs"
 759
 760     valgrind_dbg_and_supp="$(
 761         strace -e open valgrind /bin/true 2>&1 >/dev/null |
 762         perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
 763     )"
 764     image_install "$valgrind_dbg_and_supp"
 765 }
 766
 767 create_valgrind_wrapper() {
 768     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 769     ddebug "Create $valgrind_wrapper"
 770     cat >"$valgrind_wrapper" <<EOF
 771 #!/usr/bin/env bash
 772
 773 mount -t proc proc /proc
 774 exec valgrind --leak-check=full --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 775 EOF
 776     chmod 0755 "$valgrind_wrapper"
 777 }
 778
 779 create_asan_wrapper() {
 780     local asan_wrapper="$initdir/$ROOTLIBDIR/systemd-under-asan"
 781     dinfo "Create ASan wrapper as '$asan_wrapper'"
 782
 783     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 784
 785     # clang: install llvm-symbolizer to generate useful reports
 786     # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
 787     [[ "$ASAN_COMPILER" == "clang" ]] && image_install "llvm-symbolizer"
 788
 789     cat >"$asan_wrapper" <<EOF
 790 #!/usr/bin/env bash
 791
 792 set -x
 793
 794 echo "ASan RT: $ASAN_RT_PATH"
 795 if [[ ! -e "$ASAN_RT_PATH" ]]; then
 796     echo >&2 "Couldn't find ASan RT at '$ASAN_RT_PATH', can't continue"
 797     exit 1
 798 fi
 799
 800 # Suppress certain leaks reported by LSan (either in external tools or bogus
 801 # ones)
 802 # Docs: # https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer#suppressions
 803 #
 804 # - fsck is called by systemd-homed and is reporting a leak we're not interested
 805 #   in
 806 # - libLLVM is a "side effect" caused by the previous fsck leak
 807 cat >/systemd-lsan.supp <<INNER_EOF
 808 leak:/bin/fsck$
 809 leak:/sbin/fsck$
 810 leak:/lib/libLLVM
 811 INNER_EOF
 812
 813 DEFAULT_LSAN_OPTIONS=${LSAN_OPTIONS:-}:suppressions=/systemd-lsan.supp
 814 DEFAULT_ASAN_OPTIONS=${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}
 815 DEFAULT_UBSAN_OPTIONS=${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}
 816 DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS LSAN_OPTIONS=\$DEFAULT_LSAN_OPTIONS"
 817
 818 # As right now bash is the PID 1, we can't expect PATH to have a sane value.
 819 # Let's make one to prevent unexpected "<bin> not found" issues in the future
 820 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
 821
 822 mount -t proc proc /proc
 823 mount -t sysfs sysfs /sys
 824 mount -o remount,rw /
 825
 826 # A lot of services (most notably dbus) won't start without preloading libasan
 827 # See https://github.com/systemd/systemd/issues/5004
 828 DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT LD_PRELOAD=$ASAN_RT_PATH"
 829
 830 if [[ "$ASAN_COMPILER" == "clang" ]]; then
 831   # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
 832   # unnecessary for gcc & libasan, however, for clang this is crucial, as its
 833   # runtime ASan DSO is in a non-standard (library) path.
 834   echo "${ASAN_RT_PATH%/*}" >/etc/ld.so.conf.d/asan-path-override.conf
 835   ldconfig
 836 fi
 837 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
 838 echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 839 echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf
 840
 841 # ASAN and syscall filters aren't compatible with each other.
 842 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
 843
 844 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
 845 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
 846 JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 847 mkdir -p "\$JOURNALD_CONF_DIR"
 848 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 849
 850 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
 851 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
 852 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
 853 printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"
 854
 855 # 90s isn't enough for some services to finish when literally everything is run
 856 # under ASan+UBSan in containers, which, in turn, are run in VMs.
 857 # Let's limit which environments such services should be executed in.
 858 mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
 859 printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=240s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
 860
 861 # Let's override another hard-coded timeout that kicks in too early
 862 mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
 863 printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf
 864
 865 # D-Bus has troubles during system shutdown causing it to fail. Although it's
 866 # harmless, it causes unnecessary noise in the logs, so let's disable LSan's
 867 # at_exit check just for the dbus.service
 868 mkdir -p /etc/systemd/system/dbus.service.d
 869 printf "[Service]\nEnvironment=ASAN_OPTIONS=leak_check_at_exit=false\n" >/etc/systemd/system/dbus.service.d/disable-lsan.conf
 870
 871 # Some utilities run via IMPORT/RUN/PROGRAM udev directives fail because
 872 # they're uninstrumented (like dmsetup). Let's add a simple rule which sets
 873 # LD_PRELOAD to the ASan RT library to fix this.
 874 mkdir -p /etc/udev/rules.d
 875 cat >/etc/udev/rules.d/00-set-LD_PRELOAD.rules <<INNER_EOF
 876 SUBSYSTEM=="block", ENV{LD_PRELOAD}="$ASAN_RT_PATH"
 877 INNER_EOF
 878 chmod 0644 /etc/udev/rules.d/00-set-LD_PRELOAD.rules
 879
 880 # The 'mount' utility doesn't behave well under libasan, causing unexpected
 881 # fails during boot and subsequent test results check:
 882 # bash-5.0# mount -o remount,rw -v /
 883 # mount: /dev/sda1 mounted on /.
 884 # bash-5.0# echo \$?
 885 # 1
 886 # Let's workaround this by clearing the previously set LD_PRELOAD env variable,
 887 # so the libasan library is not loaded for this particular service
 888 unset_ld_preload() {
 889     local _dropin_dir="/etc/systemd/system/\$1.service.d"
 890     mkdir -p "\$_dropin_dir"
 891     printf "[Service]\nUnsetEnvironment=LD_PRELOAD\n" >"\$_dropin_dir/unset_ld_preload.conf"
 892 }
 893
 894 unset_ld_preload systemd-remount-fs
 895 unset_ld_preload testsuite-
 896
 897 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 898 exec "$ROOTLIBDIR/systemd" "\$@"
 899 EOF
 900
 901     chmod 0755 "$asan_wrapper"
 902 }
 903
 904 create_strace_wrapper() {
 905     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
 906     ddebug "Create $strace_wrapper"
 907     cat >"$strace_wrapper" <<EOF
 908 #!/usr/bin/env bash
 909
 910 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
 911 EOF
 912     chmod 0755 "$strace_wrapper"
 913 }
 914
 915 install_fs_tools() {
 916     dinfo "Install fsck"
 917     image_install /sbin/fsck*
 918     image_install -o /bin/fsck*
 919
 920     # fskc.reiserfs calls reiserfsck. so, install it
 921     image_install -o reiserfsck
 922
 923     # we use mkfs in system-repart tests
 924     image_install /sbin/mkfs.ext4
 925     image_install /sbin/mkfs.vfat
 926 }
 927
 928 install_modules() {
 929     dinfo "Install modules"
 930
 931     instmods loop
 932     instmods vfat
 933     instmods nls_ascii =nls
 934     instmods dummy
 935
 936     if get_bool "$LOOKS_LIKE_SUSE"; then
 937         instmods ext4
 938     fi
 939 }
 940
 941 install_dmevent() {
 942     instmods dm_crypt =crypto
 943     inst_binary dmeventd
 944     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
 945     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 946         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
 947         # and since buster/bionic 95-dm-notify.rules
 948         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
 949         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
 950     else
 951         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
 952     fi
 953     if get_bool "$LOOKS_LIKE_SUSE"; then
 954         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
 955     fi
 956 }
 957
 958 install_multipath() {
 959     instmods "=md" multipath
 960     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
 961     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
 962     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 963         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
 964     else
 965         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
 966     fi
 967     mkdir -p "${initdir:?}/etc/multipath"
 968
 969     local file
 970     while read -r file; do
 971         # Install libraries required by the given library
 972         inst_libs "$file"
 973         # Install the library itself and create necessary symlinks
 974         inst_library "$file"
 975     done < <(find /lib*/multipath -type f)
 976
 977     if get_bool "$LOOKS_LIKE_ARCH"; then
 978         # On Arch the multipath libraries are not linked against libgcc_s.so.1,
 979         # but it's still required at runtime
 980         inst_library "/lib64/libgcc_s.so.1"
 981     fi
 982 }
 983
 984 install_lvm() {
 985     image_install lvm
 986     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
 987     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
 988     image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
 989     image_install -o "/lib/tmpfiles.d/lvm2.conf"
 990     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 991         inst_rules 56-lvm.rules 69-lvm-metad.rules
 992     else
 993         # Support the new udev autoactivation introduced in lvm 2.03.14
 994         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
 995         if [[ -f /lib/udev/rules.d/69-dm-lvm.rules ]]; then
 996             inst_rules 11-dm-lvm.rules 69-dm-lvm.rules
 997         else
 998             image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
 999             inst_rules 11-dm-lvm.rules 69-dm-lvm-metad.rules
1000         fi
1001     fi
1002     mkdir -p "${initdir:?}/etc/lvm"
1003 }
1004
1005 install_btrfs() {
1006     instmods btrfs
1007     # Not all utilities provided by btrfs-progs are listed here; extend the list
1008     # if necessary
1009     image_install btrfs btrfstune mkfs.btrfs
1010     inst_rules 64-btrfs-dm.rules
1011 }
1012
1013 install_iscsi() {
1014     # Install both client and server side stuff by default
1015     local inst="${1:-}"
1016     local file
1017
1018     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1019     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1020     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1021         image_install iscsi-iname iscsiadm iscsid iscsistart
1022         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1023         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1024         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1025         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1026         mkdir -p "${initdir:?}/etc/iscsi"
1027         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1028         inst_simple "/etc/iscsi/initiatorname.iscsi"
1029     fi
1030
1031     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1032     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1033     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1034         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1035         image_install -o /etc/sysconfig/tgtd
1036         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1037         mkdir -p "${initdir:?}/etc/tgt"
1038         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1039         # Install perl modules required by tgt-admin
1040         #
1041         # Forgive me father for I have sinned. The monstrosity below appends
1042         # a perl snippet to the `tgt-admin` perl script on the fly, which
1043         # dumps a list of files (perl modules) required by `tgt-admin` at
1044         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1045         # passed to `inst_simple` which installs the necessary files into the image
1046         #
1047         # shellcheck disable=SC2016
1048         while read -r file; do
1049             inst_simple "$file"
1050         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1051     fi
1052 }
1053
1054 install_compiled_systemd() {
1055     dinfo "Install compiled systemd"
1056
1057     local ninja_bin
1058     ninja_bin="$(type -P ninja || type -P ninja-build)"
1059     if [[ -z "$ninja_bin" ]]; then
1060         dfatal "ninja was not found"
1061         exit 1
1062     fi
1063     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1064
1065     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1066     # find them in the same directory as the runtime data counts
1067     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1068         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1069         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1070     fi
1071 }
1072
1073 install_debian_systemd() {
1074     dinfo "Install debian systemd"
1075
1076     local files
1077
1078     while read -r deb; do
1079         files="$(dpkg-query -L "$deb" 2>/dev/null)" || continue
1080         ddebug "Install debian files from package $deb"
1081         for file in $files; do
1082             [ -e "$file" ] || continue
1083             [ -d "$file" ] && continue
1084             inst "$file"
1085         done
1086     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1087 }
1088
1089 install_suse_systemd() {
1090     local testsdir=/usr/lib/systemd/tests
1091     local pkgs
1092
1093     dinfo "Install SUSE systemd"
1094
1095     pkgs=(
1096         systemd
1097         systemd-container
1098         systemd-coredump
1099         systemd-experimental
1100         systemd-journal-remote
1101         systemd-portable
1102         udev
1103     )
1104
1105     for p in "${pkgs[@]}"; do
1106         rpm -q "$p" &>/dev/null || continue
1107
1108         ddebug "Install files from package $p"
1109         while read -r f; do
1110             [ -e "$f" ] || continue
1111             [ -d "$f" ] && continue
1112             inst "$f"
1113         done < <(rpm -ql "$p")
1114     done
1115
1116     # we only need testsdata dir as well as the unit tests (for
1117     # TEST-02-UNITTESTS) in the image.
1118     dinfo "Install unit tests and testdata directory"
1119
1120     mkdir -p "$initdir/$testsdir"
1121     cp "$testsdir"/test-* "$initdir/$testsdir/"
1122     cp -a "$testsdir/testdata" "$initdir/$testsdir/"
1123
1124     # On openSUSE, these dirs are not created at package install for now on.
1125     mkdir -p "$initdir/var/log/journal/remote"
1126 }
1127
1128 install_distro_systemd() {
1129     dinfo "Install distro systemd"
1130
1131     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1132         install_debian_systemd
1133     elif get_bool "$LOOKS_LIKE_SUSE"; then
1134         install_suse_systemd
1135     else
1136         dfatal "NO_BUILD not supported for this distro"
1137         exit 1
1138     fi
1139 }
1140
1141 install_systemd() {
1142     dinfo "Install systemd"
1143     if get_bool "$NO_BUILD"; then
1144         install_distro_systemd
1145     else
1146         install_compiled_systemd
1147     fi
1148
1149     # remove unneeded documentation
1150     rm -fr "${initdir:?}"/usr/share/{man,doc}
1151
1152     # enable debug logging in PID1
1153     echo LogLevel=debug >>"$initdir/etc/systemd/system.conf"
1154     # store coredumps in journal
1155     echo Storage=journal >>"$initdir/etc/systemd/coredump.conf"
1156     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1157     mkdir "$initdir/etc/systemd/system/user@.service.d/"
1158     echo -e "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/override.conf"
1159
1160     # When built with gcov, disable ProtectSystem= and ProtectHome in the test
1161     # images, since it prevents gcov to write the coverage reports (*.gcda files)
1162     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1163         mkdir -p "$initdir/etc/systemd/system/service.d/"
1164         echo -e "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/gcov-override.conf"
1165     fi
1166 }
1167
1168 get_ldpath() {
1169     local rpath
1170     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1171
1172     if [ -z "$rpath" ] ; then
1173         echo "$BUILD_DIR"
1174     else
1175         echo "$rpath"
1176     fi
1177 }
1178
1179 install_missing_libraries() {
1180     dinfo "Install missing libraries"
1181     # install possible missing libraries
1182     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
1183         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1184     done
1185
1186     local lib path
1187     # A number of dependencies is now optional via dlopen, so the install
1188     # script will not pick them up, since it looks at linkage.
1189     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu libfido2 libbpf libelf libdw; do
1190         ddebug "Searching for $lib via pkg-config"
1191         if pkg-config --exists "$lib"; then
1192                 path="$(pkg-config --variable=libdir "$lib")"
1193                 if [ -z "${path}" ]; then
1194                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1195                     continue
1196                 fi
1197
1198                 if ! [[ ${lib} =~ ^lib ]]; then
1199                         lib="lib${lib}"
1200                 fi
1201                 # Some pkg-config files are broken and give out the wrong paths
1202                 # (eg: libcryptsetup), so just ignore them
1203                 inst_libs "${path}/${lib}.so" || true
1204                 inst_library "${path}/${lib}.so" || true
1205         else
1206             ddebug "$lib.pc not found, skipping"
1207             continue
1208         fi
1209     done
1210 }
1211
1212 cleanup_loopdev() {
1213     if [ -n "${LOOPDEV:=}" ]; then
1214         ddebug "losetup -d $LOOPDEV"
1215         losetup -d "${LOOPDEV}"
1216         unset LOOPDEV
1217     fi
1218 }
1219
1220 trap cleanup_loopdev EXIT INT QUIT PIPE
1221
1222 create_empty_image() {
1223     if [ -z "${IMAGE_NAME:=}" ]; then
1224         echo "create_empty_image: \$IMAGE_NAME not set"
1225         exit 1
1226     fi
1227
1228     local size=500
1229     if ! get_bool "$NO_BUILD"; then
1230         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1231             size=$((size+=200))
1232         fi
1233         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1234             size=$((size+=200))
1235         fi
1236         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1237             size=$((size+=250))
1238         fi
1239     fi
1240     if ! get_bool "$STRIP_BINARIES"; then
1241         size=$((4 * size))
1242     fi
1243
1244     echo "Setting up ${IMAGE_PUBLIC:?} (${size} MB)"
1245     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1246
1247     # Create the blank file to use as a root filesystem
1248     truncate -s "${size}M" "$IMAGE_PUBLIC"
1249
1250     LOOPDEV=$(losetup --show -P -f "$IMAGE_PUBLIC")
1251     [ -b "$LOOPDEV" ] || return 1
1252     sfdisk "$LOOPDEV" <<EOF
1253 ,$((size - 50))M,L,*
1254 ,
1255 EOF
1256
1257     udevadm settle
1258
1259     local label=(-L systemd_boot)
1260     # mkfs.reiserfs doesn't know -L. so, use --label instead
1261     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1262     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p1" -q; then
1263         dfatal "Failed to mkfs -t ${FSTYPE}"
1264         exit 1
1265     fi
1266 }
1267
1268 mount_initdir() {
1269     if [ -z "${LOOPDEV:=}" ]; then
1270         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1271         LOOPDEV="$(losetup --show -P -f "$image")"
1272         [ -b "$LOOPDEV" ] || return 1
1273
1274         udevadm settle
1275     fi
1276
1277     if ! mountpoint -q "${initdir:?}"; then
1278         mkdir -p "$initdir"
1279         mount "${LOOPDEV}p1" "$initdir"
1280         TEST_SETUP_CLEANUP_ROOTDIR=1
1281     fi
1282 }
1283
1284 cleanup_initdir() {
1285     # only umount if create_empty_image_rootdir() was called to mount it
1286     get_bool "$TEST_SETUP_CLEANUP_ROOTDIR" && _umount_dir "${initdir:?}"
1287 }
1288
1289 umount_loopback() {
1290     # unmount the loopback device from all places. Otherwise we risk file
1291     # system corruption.
1292     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1293         ddebug "Unmounting all uses of $device"
1294         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1295     done
1296 }
1297
1298 create_empty_image_rootdir() {
1299     create_empty_image
1300     mount_initdir
1301 }
1302
1303 check_asan_reports() {
1304     local ret=0
1305     local root="${1:?}"
1306
1307     if get_bool "$IS_BUILT_WITH_ASAN"; then
1308         ls -l "$root"
1309         if [[ -e "$root/systemd.asan.log.1" ]]; then
1310             cat "$root/systemd.asan.log.1"
1311             ret=$((ret+1))
1312         fi
1313
1314         journald_report="$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)"
1315         if [[ -n "$journald_report" ]]; then
1316             printf "%s\n" "$journald_report"
1317             cat "$root/systemd-journald.out" || :
1318             ret=$((ret+1))
1319         fi
1320
1321         pids="$(
1322             "$JOURNALCTL" -D "$root/var/log/journal" | perl -alne '
1323                  BEGIN {
1324                      %services_to_ignore = (
1325                          "dbus-daemon" => undef,
1326                      );
1327                  }
1328                  print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
1329         )"
1330         if [[ -n "$pids" ]]; then
1331             ret=$((ret+1))
1332             for pid in $pids; do
1333                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1334             done
1335         fi
1336     fi
1337
1338     return $ret
1339 }
1340
1341 check_coverage_reports() {
1342     local root="${1:?}"
1343
1344     if get_bool "$NO_BUILD"; then
1345         return 0
1346     fi
1347     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1348         return 0
1349     fi
1350
1351     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1352         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1353     else
1354         dest="${TESTDIR:?}/coverage-info"
1355     fi
1356
1357     # Create a coverage report that will later be uploaded. Remove info about
1358     # system libraries/headers, as we don't really care about them.
1359     if [[ -f "$dest" ]]; then
1360         # If the destination report file already exists, don't overwrite it, but
1361         # dump the new report in a temporary file and then merge it with the already
1362         # present one - this usually happens when running both "parts" of a test
1363         # in one run (the qemu and the nspawn part).
1364         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1365         lcov --remove "${dest}.new" -o "${dest}.new" '/usr/include/*' '/usr/lib/*'
1366         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1367         rm -f "${dest}.new"
1368     else
1369         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}"
1370         lcov --remove "${dest}" -o "${dest}" '/usr/include/*' '/usr/lib/*'
1371     fi
1372
1373     return 0
1374 }
1375
1376 save_journal() {
1377     # Default to always saving journal
1378     local save="yes"
1379
1380     if [ "${TEST_SAVE_JOURNAL}" = "no" ]; then
1381         save="no"
1382     elif [ "${TEST_SAVE_JOURNAL}" = "fail" ] && [ "$2" = "0" ]; then
1383         save="no"
1384     fi
1385
1386     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1387         dest="${ARTIFACT_DIRECTORY}/${testname:?}.journal"
1388     else
1389         dest="${TESTDIR:?}/system.journal"
1390     fi
1391
1392     for j in "${1:?}"/*; do
1393         if get_bool "$save"; then
1394             "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $j"
1395         fi
1396
1397         if [ -n "${TEST_SHOW_JOURNAL}" ]; then
1398             echo "---- $j ----"
1399             "$JOURNALCTL" --no-pager -o short-monotonic --no-hostname --priority="${TEST_SHOW_JOURNAL}" -D "$j"
1400         fi
1401
1402         rm -r "$j"
1403     done
1404
1405     if ! get_bool "$save"; then
1406         return 0
1407     fi
1408
1409     if [ -n "${SUDO_USER}" ]; then
1410         setfacl -m "user:${SUDO_USER:?}:r-X" "$dest"*
1411     fi
1412
1413     # we want to print this sometime later, so save this in a variable
1414     JOURNAL_LIST="$(ls -l "$dest"*)"
1415 }
1416
1417 check_result_common() {
1418     local workspace="${1:?}"
1419     local ret
1420
1421     if [ -s "$workspace/failed" ]; then
1422         # Non-empty …/failed has highest priority
1423         cp -a "$workspace/failed" "${TESTDIR:?}/"
1424         if [ -n "${SUDO_USER}" ]; then
1425             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1426         fi
1427         ret=1
1428     elif get_bool "$TIMED_OUT"; then
1429         echo "(timeout)" >"${TESTDIR:?}/failed"
1430         ret=2
1431     elif [ -e "$workspace/testok" ]; then
1432         # …/testok always counts (but with lower priority than …/failed)
1433         ret=0
1434     elif [ -e "$workspace/skipped" ]; then
1435         # …/skipped always counts (a message is expected)
1436         echo "${TESTNAME:?} was skipped:"
1437         cat "$workspace/skipped"
1438         ret=0
1439     else
1440         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1441         ret=3
1442     fi
1443
1444     check_asan_reports "$workspace" || ret=4
1445
1446     check_coverage_reports "$workspace" || ret=5
1447
1448     save_journal "$workspace/var/log/journal" $ret
1449
1450     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1451         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1452     fi
1453
1454     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1455         echo -n "${TESTNAME:?}: "
1456         cat "$TESTDIR/failed"
1457     fi
1458     echo "${JOURNAL_LIST:-"No journals were saved"}"
1459
1460     return ${ret:?}
1461 }
1462
1463 check_result_nspawn() {
1464     local workspace="${1:?}"
1465     local ret
1466
1467     check_result_common "${workspace}"
1468     ret=$?
1469
1470     # Run additional test-specific checks if defined by check_result_nspawn_hook()
1471     if declare -F check_result_nspawn_hook >/dev/null; then
1472         if ! check_result_nspawn_hook; then
1473             derror "check_result_nspawn_hook() returned with EC > 0"
1474             ret=4
1475         fi
1476     fi
1477
1478     _umount_dir "${initdir:?}"
1479
1480     return $ret
1481 }
1482
1483 # can be overridden in specific test
1484 check_result_qemu() {
1485     local ret
1486     mount_initdir
1487
1488     check_result_common "${initdir:?}"
1489     ret=$?
1490
1491     _umount_dir "${initdir:?}"
1492
1493     # Run additional test-specific checks if defined by check_result_qemu_hook()
1494     if declare -F check_result_qemu_hook >/dev/null; then
1495         if ! check_result_qemu_hook; then
1496             derror "check_result_qemu_hook() returned with EC > 0"
1497             ret=4
1498         fi
1499     fi
1500
1501     return $ret
1502 }
1503
1504 check_result_nspawn_unittests() {
1505     local workspace="${1:?}"
1506     local ret=1
1507
1508     [[ -e "$workspace/testok" ]] && ret=0
1509
1510     if [[ -s "$workspace/failed" ]]; then
1511         ret=$((ret + 1))
1512         echo "=== Failed test log ==="
1513         cat "$workspace/failed"
1514     else
1515         if [[ -s "$workspace/skipped" ]]; then
1516             echo "=== Skipped test log =="
1517             cat "$workspace/skipped"
1518             # We might have only skipped tests - that should not fail the job
1519             ret=0
1520         fi
1521         if [[ -s "$workspace/testok" ]]; then
1522             echo "=== Passed tests ==="
1523             cat "$workspace/testok"
1524         fi
1525     fi
1526
1527     get_bool "${TIMED_OUT:=}" && ret=1
1528     check_coverage_reports "$workspace" || ret=5
1529
1530     save_journal "$workspace/var/log/journal" $ret
1531
1532     _umount_dir "${initdir:?}"
1533
1534     return $ret
1535 }
1536
1537 check_result_qemu_unittests() {
1538     local ret=1
1539
1540     mount_initdir
1541     [[ -e "${initdir:?}/testok" ]] && ret=0
1542
1543     if [[ -s "$initdir/failed" ]]; then
1544         ret=$((ret + 1))
1545         echo "=== Failed test log ==="
1546         cat "$initdir/failed"
1547     else
1548         if [[ -s "$initdir/skipped" ]]; then
1549             echo "=== Skipped test log =="
1550             cat "$initdir/skipped"
1551             # We might have only skipped tests - that should not fail the job
1552             ret=0
1553         fi
1554         if [[ -s "$initdir/testok" ]]; then
1555             echo "=== Passed tests ==="
1556             cat "$initdir/testok"
1557         fi
1558     fi
1559
1560     get_bool "${TIMED_OUT:=}" && ret=1
1561     check_coverage_reports "$initdir" || ret=5
1562
1563     save_journal "$initdir/var/log/journal" $ret
1564
1565     _umount_dir "$initdir"
1566
1567     return $ret
1568 }
1569
1570 strip_binaries() {
1571     dinfo "Strip binaries"
1572     if ! get_bool "$STRIP_BINARIES"; then
1573         dinfo "STRIP_BINARIES == no, keeping binaries unstripped"
1574         return 0
1575     fi
1576     while read -r bin; do
1577         strip --strip-unneeded "$bin" |& grep -vi 'file format not recognized' | ddebug || :
1578     done < <(find "${initdir:?}" -executable -not -path '*/lib/modules/*.ko' -type f)
1579 }
1580
1581 create_rc_local() {
1582     dinfo "Create rc.local"
1583     mkdir -p "${initdir:?}/etc/rc.d"
1584     cat >"$initdir/etc/rc.d/rc.local" <<EOF
1585 #!/usr/bin/env bash
1586 exit 0
1587 EOF
1588     chmod 0755 "$initdir/etc/rc.d/rc.local"
1589 }
1590
1591 install_execs() {
1592     ddebug "Install executables from the service files"
1593
1594     local pkg_config_path="${BUILD_DIR:?}/src/core/"
1595     local systemunitdir userunitdir exe
1596     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
1597     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
1598     while read -r exe; do
1599         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
1600         # also, plymouth is pulled in by rescue.service, but even there the exit code
1601         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
1602         dinfo "Attempting to install $exe (based on unit file reference)"
1603         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
1604     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
1605 }
1606
1607 generate_module_dependencies() {
1608     dinfo "Generate modules dependencies"
1609     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
1610         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
1611             dfatal "\"depmod -a $KERNEL_VER\" failed."
1612             exit 1
1613     fi
1614 }
1615
1616 install_depmod_files() {
1617     dinfo "Install depmod files"
1618     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
1619     inst "/lib/modules/$KERNEL_VER/modules.builtin"
1620 }
1621
1622 install_plymouth() {
1623     dinfo "Install plymouth"
1624     # install plymouth, if found... else remove plymouth service files
1625     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
1626     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
1627     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
1628     #         image_install plymouth plymouthd
1629     # else
1630         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
1631     # fi
1632 }
1633
1634 install_haveged() {
1635     # If haveged is installed, it's probably included in initrd and needs to be
1636     # installed in the image too.
1637     if [ -x /usr/sbin/haveged ]; then
1638         dinfo "Install haveged files"
1639         inst /usr/sbin/haveged
1640         for u in /usr/lib/systemd/system/haveged*; do
1641             inst "$u"
1642         done
1643     fi
1644 }
1645
1646 install_ld_so_conf() {
1647     dinfo "Install /etc/ld.so.conf*"
1648     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
1649     ldconfig -r "$initdir"
1650 }
1651
1652 install_testuser() {
1653     dinfo "Set up a test user"
1654     # create unprivileged user for user manager tests
1655     mkdir -p "${initdir:?}/etc/sysusers.d"
1656     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
1657 u testuser    4711     "Test User" /home/testuser
1658 EOF
1659
1660     mkdir -p "$initdir/home/testuser"
1661     chmod 0700 "$initdir/home/testuser"
1662     chown 4711:4711 "$initdir/home/testuser"
1663 }
1664
1665 install_config_files() {
1666     dinfo "Install config files"
1667     inst /etc/sysconfig/init || :
1668     inst /etc/passwd
1669     inst /etc/shadow
1670     inst_any /etc/login.defs /usr/etc/login.defs
1671     inst /etc/group
1672     inst /etc/shells
1673     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
1674     inst /etc/pam.conf || :
1675     inst_any /etc/os-release /usr/lib/os-release
1676     inst /etc/localtime
1677     # we want an empty environment
1678     : >"${initdir:?}/etc/environment"
1679     : >"$initdir/etc/machine-id"
1680     : >"$initdir/etc/resolv.conf"
1681
1682     # set the hostname
1683     echo 'H' >"$initdir/etc/hostname"
1684
1685     # let's set up just one image with the traditional verbose output
1686     if [ "${IMAGE_NAME:?}" != "basic" ]; then
1687         mkdir -p "$initdir/etc/systemd/system.conf.d"
1688         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
1689     fi
1690 }
1691
1692 install_basic_tools() {
1693     dinfo "Install basic tools"
1694     image_install "${BASICTOOLS[@]}"
1695     image_install -o sushell
1696     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
1697     image_install -o ldconfig.real
1698 }
1699
1700 install_debug_tools() {
1701     dinfo "Install debug tools"
1702     image_install -o "${DEBUGTOOLS[@]}"
1703
1704     if get_bool "$INTERACTIVE_DEBUG"; then
1705         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
1706         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
1707         mkdir -p "$getty_override"
1708         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
1709
1710         cat >"$initdir/etc/motd" <<EOF
1711 To adjust the terminal size use:
1712     export COLUMNS=xx
1713     export LINES=yy
1714 or
1715     stty cols xx rows yy
1716 EOF
1717     fi
1718 }
1719
1720 install_libnss() {
1721     dinfo "Install libnss"
1722     # install libnss_files for login
1723     local NSS_LIBS
1724     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
1725     image_install "${NSS_LIBS[@]}"
1726 }
1727
1728 install_dbus() {
1729     dinfo "Install dbus"
1730     inst "${ROOTLIBDIR:?}/system/dbus.socket"
1731
1732     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1733     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
1734         inst "$ROOTLIBDIR/system/dbus-broker.service"
1735         inst_symlink /etc/systemd/system/dbus.service
1736         inst /usr/bin/dbus-broker
1737         inst /usr/bin/dbus-broker-launch
1738     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
1739         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1740         inst "$ROOTLIBDIR/system/dbus-daemon.service"
1741         # Alias symlink
1742         inst_symlink /etc/systemd/system/dbus.service
1743     else
1744         inst "$ROOTLIBDIR/system/dbus.service"
1745     fi
1746
1747     while read -r file; do
1748         inst "$file"
1749     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
1750
1751     # setup policy for Type=dbus test
1752     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
1753     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
1754 <?xml version="1.0"?>
1755 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
1756         "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
1757 <busconfig>
1758     <policy user="root">
1759         <allow own="systemd.test.ExecStopPost"/>
1760     </policy>
1761 </busconfig>
1762 EOF
1763 }
1764
1765 install_user_dbus() {
1766     dinfo "Install user dbus"
1767     local userunitdir
1768     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
1769         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
1770         userunitdir=/usr/lib/systemd/user
1771     fi
1772
1773     inst "$userunitdir/dbus.socket"
1774     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
1775
1776     # Append the After= dependency on dbus in case it isn't already set up
1777     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
1778     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
1779 [Unit]
1780 After=dbus.service
1781 EOF
1782
1783     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1784     if [ -f "$userunitdir/dbus-broker.service" ]; then
1785         inst "$userunitdir/dbus-broker.service"
1786         inst_symlink /etc/systemd/user/dbus.service
1787     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
1788         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1789         inst "$userunitdir/dbus-daemon.service"
1790         # Alias symlink
1791         inst_symlink /etc/systemd/user/dbus.service
1792     else
1793         inst "$userunitdir/dbus.service"
1794     fi
1795 }
1796
1797 install_pam() {
1798     dinfo "Install PAM"
1799     local paths=()
1800
1801     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
1802         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
1803     else
1804         paths+=(/lib*/security)
1805     fi
1806
1807     for d in /etc/pam.d /etc/security /usr/{etc,lib}/pam.d; do
1808         [ -d "$d" ] && paths+=("$d")
1809     done
1810
1811     while read -r file; do
1812         inst "$file"
1813     done < <(find "${paths[@]}" -xtype f)
1814
1815     # pam_unix depends on unix_chkpwd.
1816     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
1817     image_install -o unix_chkpwd
1818
1819     # set empty root password for easy debugging
1820     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
1821
1822     # And make sure pam_unix will accept it by making sure that
1823     # the PAM module has the nullok option.
1824     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
1825         [ -d "$initdir/$d" ] || continue
1826         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
1827     done
1828 }
1829
1830 # shellcheck disable=SC2120
1831 install_keymaps() {
1832     dinfo "Install keymaps"
1833     # The first three paths may be deprecated.
1834     # It seems now the last two paths are used by many distributions.
1835     for i in \
1836         /usr/lib/kbd/keymaps/include/* \
1837         /usr/lib/kbd/keymaps/i386/include/* \
1838         /usr/lib/kbd/keymaps/i386/qwerty/us.* \
1839         /usr/lib/kbd/keymaps/legacy/include/* \
1840         /usr/lib/kbd/keymaps/legacy/i386/qwerty/us.*; do
1841             [[ -f "$i" ]] || continue
1842             inst "$i"
1843     done
1844
1845     # When it takes any argument, then install more keymaps.
1846     if [[ $# -gt 1 ]]; then
1847         for i in \
1848         /usr/lib/kbd/keymaps/i386/*/* \
1849         /usr/lib/kbd/keymaps/legacy/i386/*/*; do
1850             [[ -f "$i" ]] || continue
1851             inst "$i"
1852         done
1853     fi
1854 }
1855
1856 install_zoneinfo() {
1857     dinfo "Install time zones"
1858     inst_any /usr/share/zoneinfo/Asia/Seoul
1859     inst_any /usr/share/zoneinfo/Asia/Vladivostok
1860     inst_any /usr/share/zoneinfo/Australia/Sydney
1861     inst_any /usr/share/zoneinfo/Europe/Berlin
1862     inst_any /usr/share/zoneinfo/Europe/Dublin
1863     inst_any /usr/share/zoneinfo/Europe/Kiev
1864     inst_any /usr/share/zoneinfo/Pacific/Auckland
1865     inst_any /usr/share/zoneinfo/Pacific/Honolulu
1866     inst_any /usr/share/zoneinfo/CET
1867     inst_any /usr/share/zoneinfo/EET
1868     inst_any /usr/share/zoneinfo/UTC
1869 }
1870
1871 install_fonts() {
1872     dinfo "Install system fonts"
1873     for i in \
1874         /usr/lib/kbd/consolefonts/eurlatgr* \
1875         /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
1876             [[ -f "$i" ]] || continue
1877             inst "$i"
1878     done
1879 }
1880
1881 install_terminfo() {
1882     dinfo "Install terminfo files"
1883     local terminfodir
1884     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
1885         [ -f "${terminfodir}/l/linux" ] && break
1886     done
1887     image_install -o "${terminfodir}/l/linux"
1888 }
1889
1890 has_user_dbus_socket() {
1891     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
1892         return 0
1893     else
1894         echo "Per-user instances are not supported. Skipping..."
1895         return 1
1896     fi
1897 }
1898
1899 setup_nspawn_root_hook() { :;}
1900
1901 setup_nspawn_root() {
1902     if [ -z "${initdir}" ]; then
1903         dfatal "\$initdir not defined"
1904         exit 1
1905     fi
1906
1907     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
1908
1909     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
1910         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
1911         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
1912     fi
1913
1914     setup_nspawn_root_hook
1915 }
1916
1917 setup_basic_dirs() {
1918     mkdir -p "${initdir:?}/run"
1919     mkdir -p "$initdir/etc/systemd/system"
1920     mkdir -p "$initdir/var/log/journal"
1921
1922
1923     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
1924         if [ -L "/$d" ]; then
1925             inst_symlink "/$d"
1926         else
1927             inst_dir "/$d"
1928         fi
1929     done
1930
1931     ln -sfn /run "$initdir/var/run"
1932     ln -sfn /run/lock "$initdir/var/lock"
1933 }
1934
1935 mask_supporting_services() {
1936     # mask some services that we do not want to run in these tests
1937     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
1938     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
1939     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
1940     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
1941     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
1942 }
1943
1944 inst_libs() {
1945     local bin="${1:?}"
1946     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
1947     local file line
1948
1949     while read -r line; do
1950         [[ "$line" = 'not a dynamic executable' ]] && break
1951         # Ignore errors about our own stuff missing. This is most likely caused
1952         # by ldd attempting to use the unprefixed RPATH.
1953         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
1954
1955         if [[ "$line" =~ $so_regex ]]; then
1956             file="${BASH_REMATCH[1]}"
1957             [[ -e "${initdir:?}/$file" ]] && continue
1958             inst_library "$file"
1959             continue
1960         fi
1961
1962         if [[ "$line" =~ not\ found ]]; then
1963             dfatal "Missing a shared library required by $bin."
1964             dfatal "Run \"ldd $bin\" to find out what it is."
1965             dfatal "$line"
1966             dfatal "Cannot create a test image."
1967             exit 1
1968         fi
1969     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
1970 }
1971
1972 import_testdir() {
1973     # make sure we don't get a stale LOOPDEV value from old times
1974     local _LOOPDEV="${LOOPDEV:=}"
1975     # We don't want shellcheck to follow & check the $STATEFILE
1976     # shellcheck source=/dev/null
1977     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
1978     LOOPDEV="$_LOOPDEV"
1979     if [[ ! -d "$TESTDIR" ]]; then
1980         if [[ -z "$TESTDIR" ]]; then
1981             TESTDIR="$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)"
1982         else
1983             mkdir -p "$TESTDIR"
1984         fi
1985
1986         cat >"$STATEFILE" <<EOF
1987 TESTDIR="$TESTDIR"
1988 EOF
1989         export TESTDIR
1990     fi
1991
1992     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
1993     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
1994 }
1995
1996 import_initdir() {
1997     initdir="${TESTDIR:?}/root"
1998     mkdir -p "$initdir"
1999     export initdir
2000 }
2001
2002 ## @brief Converts numeric logging level to the first letter of level name.
2003 #
2004 # @param lvl Numeric logging level in range from 1 to 6.
2005 # @retval 1 if @a lvl is out of range.
2006 # @retval 0 if @a lvl is correct.
2007 # @result Echoes first letter of level name.
2008 _lvl2char() {
2009     case "$1" in
2010         1) echo F;;
2011         2) echo E;;
2012         3) echo W;;
2013         4) echo I;;
2014         5) echo D;;
2015         6) echo T;;
2016         *) return 1;;
2017     esac
2018 }
2019
2020 ## @brief Internal helper function for _do_dlog()
2021 #
2022 # @param lvl Numeric logging level.
2023 # @param msg Message.
2024 # @retval 0 It's always returned, even if logging failed.
2025 #
2026 # @note This function is not supposed to be called manually. Please use
2027 # dtrace(), ddebug(), or others instead which wrap this one.
2028 #
2029 # This function calls _do_dlog() either with parameter msg, or if
2030 # none is given, it will read standard input and will use every line as
2031 # a message.
2032 #
2033 # This enables:
2034 # dwarn "This is a warning"
2035 # echo "This is a warning" | dwarn
2036 LOG_LEVEL="${LOG_LEVEL:-4}"
2037
2038 dlog() {
2039     local lvl lvlc
2040
2041     [ -z "$LOG_LEVEL" ] && return 0
2042     lvl="${1:?}"; shift
2043     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2044     lvlc="$(_lvl2char "$lvl")" || return 0
2045
2046     if [ $# -ge 1 ]; then
2047         echo "$lvlc: $*"
2048     else
2049         while read -r line; do
2050             echo "$lvlc: " "$line"
2051         done
2052     fi
2053 }
2054
2055 ## @brief Logs message at TRACE level (6)
2056 #
2057 # @param msg Message.
2058 # @retval 0 It's always returned, even if logging failed.
2059 dtrace() {
2060     set +x
2061     dlog 6 "$@"
2062     if get_bool "${debug:=}"; then
2063         set -x
2064     fi
2065 }
2066
2067 ## @brief Logs message at DEBUG level (5)
2068 #
2069 # @param msg Message.
2070 # @retval 0 It's always returned, even if logging failed.
2071 ddebug() {
2072     dlog 5 "$@"
2073 }
2074
2075 ## @brief Logs message at INFO level (4)
2076 #
2077 # @param msg Message.
2078 # @retval 0 It's always returned, even if logging failed.
2079 dinfo() {
2080     set +x
2081     dlog 4 "$@"
2082     if get_bool "${debug:=}"; then
2083         set -x
2084     fi
2085 }
2086
2087 ## @brief Logs message at WARN level (3)
2088 #
2089 # @param msg Message.
2090 # @retval 0 It's always returned, even if logging failed.
2091 dwarn() {
2092     set +x
2093     dlog 3 "$@"
2094     if get_bool "${debug:=}"; then
2095         set -x
2096     fi
2097 }
2098
2099 ## @brief Logs message at ERROR level (2)
2100 #
2101 # @param msg Message.
2102 # @retval 0 It's always returned, even if logging failed.
2103 derror() {
2104     dlog 2 "$@"
2105 }
2106
2107 ## @brief Logs message at FATAL level (1)
2108 #
2109 # @param msg Message.
2110 # @retval 0 It's always returned, even if logging failed.
2111 dfatal() {
2112     set +x
2113     dlog 1 "$@"
2114     if get_bool "${debug:=}"; then
2115         set -x
2116     fi
2117 }
2118
2119
2120 # Generic substring function.  If $2 is in $1, return 0.
2121 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2122
2123 # normalize_path <path>
2124 # Prints the normalized path, where it removes any duplicated
2125 # and trailing slashes.
2126 # Example:
2127 # $ normalize_path ///test/test//
2128 # /test/test
2129 normalize_path() {
2130     shopt -q -s extglob
2131     set -- "${1//+(\/)//}"
2132     shopt -q -u extglob
2133     echo "${1%/}"
2134 }
2135
2136 # convert_abs_rel <from> <to>
2137 # Prints the relative path, when creating a symlink to <to> from <from>.
2138 # Example:
2139 # $ convert_abs_rel /usr/bin/test /bin/test-2
2140 # ../../bin/test-2
2141 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2142 convert_abs_rel() {
2143     local __current __absolute __abssize __cursize __newpath
2144     local -i __i __level
2145
2146     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2147
2148     # corner case #1 - self looping link
2149     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2150
2151     # corner case #2 - own dir link
2152     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2153
2154     IFS="/" read -ra __current <<< "$1"
2155     IFS="/" read -ra __absolute <<< "$2"
2156
2157     __abssize=${#__absolute[@]}
2158     __cursize=${#__current[@]}
2159
2160     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]
2161     do
2162         (( __level++ ))
2163         if (( __level > __abssize || __level > __cursize ))
2164         then
2165             break
2166         fi
2167     done
2168
2169     for ((__i = __level; __i < __cursize-1; __i++))
2170     do
2171         if ((__i > __level))
2172         then
2173             __newpath=$__newpath"/"
2174         fi
2175         __newpath=$__newpath".."
2176     done
2177
2178     for ((__i = __level; __i < __abssize; __i++))
2179     do
2180         if [[ -n $__newpath ]]
2181         then
2182             __newpath=$__newpath"/"
2183         fi
2184         __newpath=$__newpath${__absolute[__i]}
2185     done
2186
2187     echo "$__newpath"
2188 }
2189
2190
2191 # Install a directory, keeping symlinks as on the original system.
2192 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2193 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2194 # and a symlink ${initdir}/lib -> lib64.
2195 inst_dir() {
2196     local dir="${1:?}"
2197     local part="${dir%/*}"
2198     local file
2199
2200     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2201
2202     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2203         dir="$part $dir"
2204         part="${part%/*}"
2205     done
2206
2207     # iterate over parent directories
2208     for file in $dir; do
2209         [[ -e "${initdir}/$file" ]] && continue
2210         if [[ -L $file ]]; then
2211             inst_symlink "$file"
2212         else
2213             # create directory
2214             mkdir -m 0755 "${initdir}/$file" || return 1
2215             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2216             chmod u+w "${initdir}/$file"
2217         fi
2218     done
2219 }
2220
2221 # $1 = file to copy to ramdisk
2222 # $2 (optional) Name for the file on the ramdisk
2223 # Location of the image dir is assumed to be $initdir
2224 # We never overwrite the target if it exists.
2225 inst_simple() {
2226     [[ -f "${1:?}" ]] || return 1
2227     strstr "$1" "/" || return 1
2228
2229     local src="$1"
2230     local target="${2:-$1}"
2231     if ! [[ -d ${initdir:?}/$target ]]; then
2232         [[ -e ${initdir}/$target ]] && return 0
2233         [[ -L ${initdir}/$target ]] && return 0
2234         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2235     fi
2236     # install checksum files also
2237     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2238         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2239     fi
2240     ddebug "Installing $src"
2241     cp --sparse=always -pfL "$src" "${initdir}/$target"
2242 }
2243
2244 # find symlinks linked to given library file
2245 # $1 = library file
2246 # Function searches for symlinks by stripping version numbers appended to
2247 # library filename, checks if it points to the same target and finally
2248 # prints the list of symlinks to stdout.
2249 #
2250 # Example:
2251 # rev_lib_symlinks libfoo.so.8.1
2252 # output: libfoo.so.8 libfoo.so
2253 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2254 rev_lib_symlinks() {
2255     local fn="${1:?}"
2256     local links=""
2257     local orig
2258     orig="$(readlink -f "$1")"
2259
2260     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2261
2262     until [[ "${fn##*.}" == so ]]; do
2263         fn="${fn%.*}"
2264         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2265     done
2266
2267     echo "${links}"
2268 }
2269
2270 # Same as above, but specialized to handle dynamic libraries.
2271 # It handles making symlinks according to how the original library
2272 # is referenced.
2273 inst_library() {
2274     local src="${1:?}"
2275     local dest="${2:-$1}"
2276     local reallib symlink
2277
2278     strstr "$1" "/" || return 1
2279     [[ -e ${initdir:?}/$dest ]] && return 0
2280     if [[ -L $src ]]; then
2281         # install checksum files also
2282         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2283             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2284         fi
2285         reallib="$(readlink -f "$src")"
2286         inst_simple "$reallib" "$reallib"
2287         inst_dir "${dest%/*}"
2288         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2289         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2290     else
2291         inst_simple "$src" "$dest"
2292     fi
2293
2294     # Create additional symlinks.  See rev_symlinks description.
2295     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2296         if [[ ! -e "$initdir/$symlink" ]]; then
2297             ddebug "Creating extra symlink: $symlink"
2298             inst_symlink "$symlink"
2299         fi
2300     done
2301 }
2302
2303 # find a binary.  If we were not passed the full path directly,
2304 # search in the usual places to find the binary.
2305 find_binary() {
2306     local bin="${1:?}"
2307     if [[ -z ${bin##/*} ]]; then
2308         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2309             echo "$bin"
2310             return 0
2311         fi
2312     fi
2313
2314     type -P "$bin"
2315 }
2316
2317 # Same as above, but specialized to install binary executables.
2318 # Install binary executable, and all shared library dependencies, if any.
2319 inst_binary() {
2320     local bin="${1:?}"
2321     local path target
2322
2323     # In certain cases we might attempt to install a binary which is already
2324     # present in the test image, yet it's missing from the host system.
2325     # In such cases, let's check if the binary indeed exists in the image
2326     # before doing any other checks. If it does, immediately return with
2327     # success.
2328     if [[ $# -eq 1 ]]; then
2329         for path in "" bin sbin usr/bin usr/sbin; do
2330             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2331         done
2332     fi
2333
2334     bin="$(find_binary "$bin")" || return 1
2335     target="${2:-$bin}"
2336     [[ -e "${initdir:?}/$target" ]] && return 0
2337     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2338
2339     local file line
2340     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2341     # I love bash!
2342     while read -r line; do
2343         [[ "$line" = 'not a dynamic executable' ]] && break
2344
2345         # Ignore errors about our own stuff missing. This is most likely caused
2346         # by ldd attempting to use the unprefixed RPATH.
2347         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2348
2349         if [[ "$line" =~ $so_regex ]]; then
2350             file="${BASH_REMATCH[1]}"
2351             [[ -e "${initdir}/$file" ]] && continue
2352             inst_library "$file"
2353             continue
2354         fi
2355
2356         if [[ "$line" =~ not\ found ]]; then
2357             dfatal "Missing a shared library required by $bin."
2358             dfatal "Run \"ldd $bin\" to find out what it is."
2359             dfatal "$line"
2360             dfatal "Cannot create a test image."
2361             exit 1
2362         fi
2363     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2364     inst_simple "$bin" "$target"
2365 }
2366
2367 # same as above, except for shell scripts.
2368 # If your shell script does not start with shebang, it is not a shell script.
2369 inst_script() {
2370     local bin line shebang_regex
2371     bin="$(find_binary "${1:?}")" || return 1
2372     shift
2373
2374     read -r -n 80 line <"$bin"
2375     # If debug is set, clean unprintable chars to prevent messing up the term
2376     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2377     shebang_regex='(#! *)(/[^ ]+).*'
2378     [[ "$line" =~ $shebang_regex ]] || return 1
2379     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2380 }
2381
2382 # same as above, but specialized for symlinks
2383 inst_symlink() {
2384     local src="${1:?}"
2385     local target="${2:-$src}"
2386     local realsrc
2387
2388     strstr "$src" "/" || return 1
2389     [[ -L "$src" ]] || return 1
2390     [[ -L "${initdir:?}/$target" ]] && return 0
2391     realsrc="$(readlink -f "$src")"
2392     if ! [[ -e "$initdir/$realsrc" ]]; then
2393         if [[ -d "$realsrc" ]]; then
2394             inst_dir "$realsrc"
2395         else
2396             inst "$realsrc"
2397         fi
2398     fi
2399     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
2400     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
2401     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
2402 }
2403
2404 # attempt to install any programs specified in a udev rule
2405 inst_rule_programs() {
2406     local rule="${1:?}"
2407     local prog bin
2408
2409     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
2410         if [ -x "/lib/udev/$prog" ]; then
2411             bin="/lib/udev/$prog"
2412         else
2413             if ! bin="$(find_binary "$prog")"; then
2414                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
2415                 continue
2416             fi
2417         fi
2418
2419         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
2420         image_install "$bin"
2421     done
2422 }
2423
2424 # udev rules always get installed in the same place, so
2425 # create a function to install them to make life simpler.
2426 inst_rules() {
2427     local target=/etc/udev/rules.d
2428     local found rule
2429
2430     inst_dir "/lib/udev/rules.d"
2431     inst_dir "$target"
2432     for rule in "$@"; do
2433         if [ "${rule#/}" = "$rule" ]; then
2434             for r in /lib/udev/rules.d /etc/udev/rules.d; do
2435                 if [[ -f "$r/$rule" ]]; then
2436                     found="$r/$rule"
2437                     inst_simple "$found"
2438                     inst_rule_programs "$found"
2439                 fi
2440             done
2441         fi
2442         for r in '' ./; do
2443             if [[ -f "${r}${rule}" ]]; then
2444                 found="${r}${rule}"
2445                 inst_simple "$found" "$target/${found##*/}"
2446                 inst_rule_programs "$found"
2447             fi
2448         done
2449         [[ $found ]] || dinfo "Skipping udev rule: $rule"
2450         found=
2451     done
2452 }
2453
2454 # general purpose installation function
2455 # Same args as above.
2456 inst() {
2457     case $# in
2458         1) ;;
2459         2)
2460             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
2461             [[ "$initdir" = "$2" ]] && set "$1"
2462             ;;
2463         3)
2464             [[ -z "$initdir" ]] && export initdir="$2"
2465             set "$1" "$3"
2466             ;;
2467         *)
2468             dfatal "inst only takes 1 or 2 or 3 arguments"
2469             exit 1
2470             ;;
2471     esac
2472
2473     local fun
2474     for fun in inst_symlink inst_script inst_binary inst_simple; do
2475         "$fun" "$@" && return 0
2476     done
2477
2478     dwarn "Failed to install '$1'"
2479     return 1
2480 }
2481
2482 # install any of listed files
2483 #
2484 # If first argument is '-d' and second some destination path, first accessible
2485 # source is installed into this path, otherwise it will installed in the same
2486 # path as source.  If none of listed files was installed, function return 1.
2487 # On first successful installation it returns with 0 status.
2488 #
2489 # Example:
2490 #
2491 # inst_any -d /bin/foo /bin/bar /bin/baz
2492 #
2493 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
2494 # initramfs.
2495 inst_any() {
2496     local dest file
2497
2498     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
2499
2500     for file in "$@"; do
2501         if [[ -e "$file" ]]; then
2502             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
2503             inst "$file" && return 0
2504         fi
2505     done
2506
2507     return 1
2508 }
2509
2510 # image_install [-o ] <file> [<file> ... ]
2511 # Install <file> to the test image
2512 # -o optionally install the <file> and don't fail, if it is not there
2513 image_install() {
2514     local optional=no
2515     local prog="${1:?}"
2516
2517     if [[ "$prog" = '-o' ]]; then
2518         optional=yes
2519         shift
2520     fi
2521
2522     for prog in "$@"; do
2523         if ! inst "$prog" ; then
2524             if get_bool "$optional"; then
2525                 dinfo "Skipping program $prog as it cannot be found and is" \
2526                     "flagged to be optional"
2527             else
2528                 dfatal "Failed to install $prog"
2529                 exit 1
2530             fi
2531         fi
2532     done
2533 }
2534
2535 # Install a single kernel module along with any firmware it may require.
2536 # $1 = full path to kernel module to install
2537 install_kmod_with_fw() {
2538     local module="${1:?}"
2539     # no need to go further if the module is already installed
2540     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
2541     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
2542
2543     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
2544
2545     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
2546
2547     local modname="${module##*/}"
2548     local fwdir found fw
2549     modname="${modname%.ko*}"
2550
2551     while read -r fw; do
2552         found=
2553         for fwdir in /lib/firmware/updates /lib/firmware; do
2554             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
2555                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
2556                 found=yes
2557             fi
2558         done
2559         if ! get_bool "$found"; then
2560             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
2561                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
2562                     "\"${modname}.ko\""
2563             else
2564                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
2565                     "\"${modname}.ko\""
2566             fi
2567         fi
2568     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
2569     return 0
2570 }
2571
2572 # Do something with all the dependencies of a kernel module.
2573 # Note that kernel modules depend on themselves using the technique we use
2574 # $1 = function to call for each dependency we find
2575 #      It will be passed the full path to the found kernel module
2576 # $2 = module to get dependencies for
2577 # rest of args = arguments to modprobe
2578 for_each_kmod_dep() {
2579     local func="${1:?}"
2580     local kmod="${2:?}"
2581     local found=0
2582     local cmd modpath
2583     shift 2
2584
2585     while read -r cmd modpath _; do
2586         [[ "$cmd" = insmod ]] || continue
2587         "$func" "$modpath" || return $?
2588         found=1
2589     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
2590
2591     ! get_bool "$found" && return 1
2592     return 0
2593 }
2594
2595 # instmods [-c] <kernel module> [<kernel module> ... ]
2596 # instmods [-c] <kernel subsystem>
2597 # install kernel modules along with all their dependencies.
2598 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
2599 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
2600 instmods() {
2601     local check=no
2602     if [[ $# -ge 0 && "$1" = '-c' ]]; then
2603         check=yes
2604         shift
2605     fi
2606
2607     inst1mod() {
2608         local mod="${1:?}"
2609         local ret=0
2610         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
2611
2612         case "$mod" in
2613             =*)
2614                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
2615                     (
2616                         [[ "$mpargs" ]] && echo "$mpargs"
2617                         cat "${mod_dir}/modules.${mod#=}"
2618                     ) | instmods
2619                 else
2620                     (
2621                         [[ "$mpargs" ]] && echo "$mpargs"
2622                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
2623                     ) | instmods
2624                 fi
2625                 ;;
2626             --*)
2627                 mpargs+=" $mod"
2628                 ;;
2629             i2o_scsi)
2630                 # Do not load this diagnostic-only module
2631                 return
2632                 ;;
2633             *)
2634                 mod=${mod##*/}
2635                 # if we are already installed, skip this module and go on
2636                 # to the next one.
2637                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
2638
2639                 # We use '-d' option in modprobe only if modules prefix path
2640                 # differs from default '/'.  This allows us to use Dracut with
2641                 # old version of modprobe which doesn't have '-d' option.
2642                 local mod_dirname=${mod_dir%%/lib/modules/*}
2643                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
2644
2645                 # ok, load the module, all its dependencies, and any firmware
2646                 # it may require
2647                 for_each_kmod_dep install_kmod_with_fw "$mod" \
2648                     --set-version "$KERNEL_VER" \
2649                     ${mod_dirname:+"$mod_dirname"} \
2650                     ${mpargs:+"$mpargs"}
2651                 ((ret+=$?))
2652                 ;;
2653         esac
2654         return "$ret"
2655     }
2656
2657     local mod mpargs
2658
2659     if [[ $# -eq 0 ]]; then  # filenames from stdin
2660         while read -r mod; do
2661             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2662                 dfatal "Failed to install $mod"
2663                 return 1
2664             fi
2665         done
2666     fi
2667
2668     for mod in "$@"; do # filenames as arguments
2669         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2670             dfatal "Failed to install $mod"
2671             return 1
2672         fi
2673     done
2674
2675     return 0
2676 }
2677
2678 _umount_dir() {
2679     local mountpoint="${1:?}"
2680     if mountpoint -q "$mountpoint"; then
2681         ddebug "umount $mountpoint"
2682         umount "$mountpoint"
2683     fi
2684 }
2685
2686 # can be overridden in specific test
2687 test_setup_cleanup() {
2688     cleanup_initdir
2689 }
2690
2691 _test_cleanup() {
2692     # (post-test) cleanup should always ignore failure and cleanup as much as possible
2693     (
2694         set +e
2695         [[ -n "$initdir" ]] && _umount_dir "$initdir"
2696         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
2697         # If multiple setups/cleans are ran in parallel, this can cause a race
2698         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
2699             rm -vf "${IMAGESTATEDIR}/default.img"
2700         fi
2701         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
2702         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
2703     ) || :
2704 }
2705
2706 # can be overridden in specific test
2707 test_cleanup() {
2708     _test_cleanup
2709 }
2710
2711 test_cleanup_again() {
2712     [ -n "$TESTDIR" ] || return
2713     rm -rf "$TESTDIR/unprivileged-nspawn-root"
2714     [[ -n "$initdir" ]] && _umount_dir "$initdir"
2715 }
2716
2717 test_create_image() {
2718     create_empty_image_rootdir
2719
2720     # Create what will eventually be our root filesystem onto an overlay
2721     (
2722         LOG_LEVEL=5
2723         setup_basic_environment
2724     )
2725 }
2726
2727 test_setup() {
2728     if get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
2729             command -v meson >/dev/null && \
2730             [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
2731         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
2732         exit 1
2733     fi
2734
2735     if [ -e "${IMAGE_PRIVATE:?}" ]; then
2736         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
2737         mount_initdir
2738     else
2739         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
2740             # default.img is the base that every test uses and optionally appends to
2741             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
2742                 # Create the backing public image, but then completely unmount
2743                 # it and drop the loopback device responsible for it, since we're
2744                 # going to symlink/copy the image and mount it again from
2745                 # elsewhere.
2746                 local image_old="${IMAGE_PUBLIC}"
2747                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
2748                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
2749                 fi
2750                 test_create_image
2751                 test_setup_cleanup
2752                 umount_loopback
2753                 cleanup_loopdev
2754                 IMAGE_PUBLIC="${image_old}"
2755             fi
2756             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
2757                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
2758             fi
2759         fi
2760
2761         local hook_defined
2762         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
2763
2764         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
2765         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
2766             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
2767         else
2768             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
2769         fi
2770
2771         mount_initdir
2772         # We want to test all services in TEST-01-BASIC, but mask them in
2773         # all other tests
2774         if [[ "${TESTID:?}" != "01" ]]; then
2775             dinfo "Masking supporting services"
2776             mask_supporting_services
2777         fi
2778
2779         if get_bool "$hook_defined"; then
2780             test_append_files "${initdir:?}"
2781         fi
2782     fi
2783
2784     setup_nspawn_root
2785 }
2786
2787 test_run() {
2788     local test_id="${1:?}"
2789     mount_initdir
2790
2791     if ! get_bool "${TEST_NO_QEMU:=}"; then
2792         if run_qemu "$test_id"; then
2793             check_result_qemu || { echo "QEMU test failed"; return 1; }
2794         else
2795             dwarn "can't run QEMU, skipping"
2796         fi
2797     fi
2798     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
2799         mount_initdir
2800         if run_nspawn "${initdir:?}" "$test_id"; then
2801             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
2802         else
2803             dwarn "can't run systemd-nspawn, skipping"
2804         fi
2805
2806         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
2807             dir="$TESTDIR/unprivileged-nspawn-root"
2808             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
2809                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
2810             else
2811                 dwarn "can't run systemd-nspawn, skipping"
2812             fi
2813         fi
2814     fi
2815     return 0
2816 }
2817
2818 do_test() {
2819     if [[ $UID != "0" ]]; then
2820         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
2821         exit 0
2822     fi
2823
2824     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
2825         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both QEMU and nspawn disabled" >&2
2826         exit 0
2827     fi
2828
2829     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
2830         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: QEMU-only tests requested" >&2
2831         exit 0
2832     fi
2833
2834     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
2835         TEST_NO_QEMU=1
2836     fi
2837
2838     # Detect lib paths
2839     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
2840         [[ -d $libdir ]] && libdirs+=" $libdir" && break
2841     done
2842
2843     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
2844         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
2845     done
2846
2847     mkdir -p "$STATEDIR"
2848
2849     import_testdir
2850     import_initdir
2851
2852     if [ -n "${SUDO_USER}" ]; then
2853         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
2854         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
2855     fi
2856
2857     testname="$(basename "$PWD")"
2858
2859     while (($# > 0)); do
2860         case $1 in
2861             --run)
2862                 echo "${testname} RUN: $TEST_DESCRIPTION"
2863                 test_run "$TESTID"
2864                 ret=$?
2865                 if [ $ret -eq 0 ]; then
2866                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
2867                 else
2868                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
2869                 fi
2870                 exit $ret
2871                 ;;
2872             --setup)
2873                 echo "${testname} SETUP: $TEST_DESCRIPTION"
2874                 test_setup
2875                 test_setup_cleanup
2876                 ;;
2877             --clean)
2878                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
2879                 test_cleanup
2880                 ;;
2881             --clean-again)
2882                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
2883                 test_cleanup_again
2884                 ;;
2885             --all)
2886                 ret=0
2887                 echo -n "${testname}: $TEST_DESCRIPTION "
2888                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
2889                 # and loop devices will leak
2890                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
2891                 if [ $ret -eq 0 ]; then
2892                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
2893                 fi
2894                 if [ $ret -eq 0 ]; then
2895                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
2896                 fi
2897                 test_cleanup
2898                 if [ $ret -eq 0 ]; then
2899                     rm "$TESTLOG"
2900                     echo "[OK]"
2901                 else
2902                     echo "[FAILED]"
2903                     echo "see $TESTLOG"
2904                 fi
2905                 exit $ret
2906                 ;;
2907             *)
2908                 break
2909                 ;;
2910         esac
2911         shift
2912     done
2913 }