test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  18 export PATH
  19
  20 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  21 # shellcheck source=/dev/null
  22 source "$os_release"
  23 [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=""
  24 [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=""
  25 [[ " $ID_LIKE " = *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=""
  26 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  27 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  28 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  29 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  30 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  31 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  32 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  33 # Note that defining a different IMAGE_NAME in a test setup script will only result
  34 # in default.img being copied and renamed. It can then be extended by defining
  35 # a test_append_files() function. The $1 parameter will be the root directory.
  36 # To force creating a new image from scratch (eg: to encrypt it), also define
  37 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  38 IMAGE_NAME=${IMAGE_NAME:-default}
  39 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  40 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  41 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  42 LOOPDEV=
  43
  44 # Simple wrapper to unify boolean checks.
  45 # Note: this function needs to stay near the top of the file, so we can use it
  46 #       in code in the outermost scope.
  47 get_bool() {
  48     # Make the value lowercase to make the regex matching simpler
  49     local _bool="${1,,}"
  50
  51     # Consider empty value as "false"
  52     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  53         return 1
  54     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  55         return 0
  56     else
  57         echo >&2 "Value '$_bool' is not a valid boolean value"
  58         exit 1
  59     fi
  60 }
  61
  62 # Since in Bash we can have only one handler per signal, let's overcome this
  63 # limitation by having one global handler for the EXIT signal which executes
  64 # all registered handlers
  65 _AT_EXIT_HANDLERS=()
  66 _at_exit() {
  67     set +e
  68
  69     # Run the EXIT handlers in reverse order
  70     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  71         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  72         eval "${_AT_EXIT_HANDLERS[$i]}"
  73     done
  74 }
  75
  76 trap _at_exit EXIT
  77
  78 add_at_exit_handler() {
  79     _AT_EXIT_HANDLERS+=("${1:?}")
  80 }
  81
  82 # Decide if we can (and want to) run qemu with KVM acceleration.
  83 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  84 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  85 # running under KVM. If these conditions are met, enable KVM (and possibly
  86 # nested KVM), otherwise disable it.
  87 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  88     QEMU_KVM=yes
  89 else
  90     QEMU_KVM=no
  91 fi
  92
  93 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  94     echo "WARNING! Cannot determine libdir from pkg-config, assuming /usr/lib/systemd" >&2
  95     ROOTLIBDIR=/usr/lib/systemd
  96 fi
  97
  98 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
  99 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 100 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 101 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 102 # These variables are used by test scripts
 103 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR
 104
 105 TOOLS_DIR="$SOURCE_DIR/tools"
 106 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 107 if get_bool "${NO_BUILD:=}"; then
 108     BUILD_DIR="$SOURCE_DIR"
 109 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 110     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 111     exit 1
 112 fi
 113
 114 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 115 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 116 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 117 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 118 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 119 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 120 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 121
 122 TESTFILE="${BASH_SOURCE[1]}"
 123 if [ -z "$TESTFILE" ]; then
 124     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 125     exit 1
 126 fi
 127 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 128
 129 WORKDIR="/var/tmp/systemd-tests"
 130 if get_bool "${NO_BUILD:=}"; then
 131     STATEDIR="$WORKDIR/$TESTNAME"
 132 else
 133     STATEDIR="$BUILD_DIR/test/$TESTNAME"
 134 fi
 135
 136 STATEFILE="$STATEDIR/.testdir"
 137 IMAGESTATEDIR="$STATEDIR/.."
 138 TESTLOG="$STATEDIR/test.log"
 139
 140 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 141     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 142     exit 1
 143 fi
 144 TESTID="${BASH_REMATCH[1]:?}"
 145
 146 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 147     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 148     exit 1
 149 fi
 150
 151 BASICTOOLS=(
 152     awk
 153     base64
 154     basename
 155     bash
 156     capsh
 157     cat
 158     chmod
 159     chown
 160     chroot
 161     cmp
 162     cryptsetup
 163     cut
 164     date
 165     dd
 166     diff
 167     dirname
 168     dmsetup
 169     echo
 170     env
 171     false
 172     flock
 173     getconf
 174     getent
 175     getfacl
 176     getfattr
 177     grep
 178     gunzip
 179     gzip
 180     head
 181     ionice
 182     ip
 183     jq
 184     killall
 185     ldd
 186     ln
 187     loadkeys
 188     login
 189     losetup
 190     lsattr
 191     lz4cat
 192     mkfifo
 193     mknod
 194     mktemp
 195     modprobe
 196     mount
 197     mountpoint
 198     mv
 199     nc
 200     nproc
 201     pkill
 202     readlink
 203     realpath
 204     rev
 205     rm
 206     rmdir
 207     rmmod
 208     script
 209     sed
 210     seq
 211     setfacl
 212     setfattr
 213     setfont
 214     setpriv
 215     setsid
 216     sfdisk
 217     sh
 218     sleep
 219     stat
 220     stty
 221     su
 222     sulogin
 223     sysctl
 224     tail
 225     tar
 226     tee
 227     test
 228     timeout
 229     touch
 230     tr
 231     true
 232     truncate
 233     umount
 234     uname
 235     unshare
 236     useradd
 237     userdel
 238     wc
 239     xargs
 240     xzcat
 241 )
 242
 243 DEBUGTOOLS=(
 244     cp
 245     df
 246     dhclient
 247     dmesg
 248     du
 249     find
 250     free
 251     grep
 252     hostname
 253     id
 254     less
 255     ln
 256     ls
 257     mkdir
 258     ping
 259     ps
 260     route
 261     sort
 262     strace
 263     tty
 264     vi
 265     /usr/libexec/vi
 266 )
 267
 268 is_built_with_asan() {
 269     local _bin="${1:?}"
 270
 271     if ! type -P objdump >/dev/null; then
 272         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
 273         return 1
 274     fi
 275
 276     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 277     local _asan_calls
 278     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 279     if ((_asan_calls < 1000)); then
 280         return 1
 281     else
 282         return 0
 283     fi
 284 }
 285
 286 is_built_with_coverage() {
 287     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 288         return 1
 289     fi
 290
 291     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 292 }
 293
 294 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 295 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 296
 297 if get_bool "$IS_BUILT_WITH_ASAN"; then
 298     PATH_TO_INIT="$ROOTLIBDIR/systemd-under-asan"
 299     QEMU_MEM="${QEMU_MEM:-2G}"
 300     QEMU_SMP="${QEMU_SMP:-4}"
 301
 302     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 303     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 304         ASAN_COMPILER=gcc
 305         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 306     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 307         ASAN_COMPILER=clang
 308         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 309
 310         # As clang's ASan DSO is usually in a non-standard path, let's check if
 311         # the environment is set accordingly. If not, warn the user and exit.
 312         # We're not setting the LD_LIBRARY_PATH automagically here, because
 313         # user should encounter (and fix) the same issue when running the unit
 314         # tests (meson test)
 315         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 316             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 317             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 318             exit 1
 319         fi
 320     else
 321         echo >&2 "systemd is not linked against the ASan DSO"
 322         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 323         exit 1
 324     fi
 325
 326     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 327 fi
 328
 329 test_require_bin() {
 330     local bin
 331
 332     for bin in "$@"; do
 333         if ! command -v "$bin" >/dev/null; then
 334             echo "Required binary $bin not available, skipping the test"
 335             exit 0
 336         fi
 337     done
 338 }
 339
 340 find_qemu_bin() {
 341     QEMU_BIN="${QEMU_BIN:-""}"
 342     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 343     if get_bool "$QEMU_KVM"; then
 344         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 345     fi
 346
 347     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 348     case $ARCH in
 349     x86_64)
 350         # QEMU's own build system calls it qemu-system-x86_64
 351         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 352         ;;
 353     i*86)
 354         # new i386 version of QEMU
 355         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 356
 357         # i386 version of QEMU
 358         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 359         ;;
 360     ppc64*)
 361         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 362         ;;
 363     esac
 364
 365     if [[ ! -e "$QEMU_BIN" ]]; then
 366         echo "Could not find a suitable qemu binary" >&2
 367         return 1
 368     fi
 369 }
 370
 371 qemu_setup_swtpm_socket() {
 372     local pid state_dir tpm_device
 373
 374     if ! tpm_device="$(qemu_get_tpm_device)"; then
 375         dinfo "Found QEMU version is too old for TPM2 on ppc64le"
 376         exit 0
 377     fi
 378
 379     state_dir="$(mktemp -d)"
 380     swtpm socket --tpm2 --tpmstate dir="$state_dir" --ctrl type=unixio,path="$state_dir/sock" &
 381     pid=$!
 382     if ! kill -0 "$pid"; then
 383         derror "Failed to start swtpm"
 384         return 1
 385     fi
 386
 387     if ! timeout 5 bash -c "until [[ -S $state_dir/sock ]]; do sleep .5; done"; then
 388         derror "Failed to setup swtpm socket"
 389         return 1
 390     fi
 391
 392     dinfo "Started swtpm as PID $pid with state dir $state_dir"
 393
 394     add_at_exit_handler "kill -TERM $pid 2>/dev/null; rm -rf '$state_dir'"
 395
 396     QEMU_OPTIONS+=" -chardev socket,id=chrtpm,path=$state_dir/sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device $tpm_device,tpmdev=tpm0"
 397     dinfo "Configured emulated TPM2 device $tpm_device"
 398
 399     return 0
 400 }
 401
 402 qemu_get_tpm_device() {
 403     local tpm_device="tpm-tis"
 404
 405     if [[ "$(uname -m)" == "ppc64le" ]]; then
 406         # tpm-spapr support was introduced in qemu 5.0.0
 407         if ! qemu_min_version "5.0.0"; then
 408             return 1
 409         fi
 410
 411         tpm_device="tpm-spapr"
 412     fi
 413
 414     echo "$tpm_device"
 415     return 0
 416 }
 417
 418 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 419 # returns 0 if newer or equal
 420 # returns 1 if older
 421 # returns 2 if failing
 422 qemu_min_version() {
 423     find_qemu_bin || return 2
 424
 425     # get version from binary
 426     local qemu_ver
 427     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 428
 429     # Check version string format
 430     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 431     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 432
 433     # compare as last command to return that value
 434     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 435 }
 436
 437 # Pads a file to multiple of 4 bytes
 438 pad4_file() {
 439     local size
 440     size=$(stat -c "%s" "$1")
 441     local padded
 442     padded=$((((size + 3) / 4) * 4))
 443     truncate -s "$padded" "$1"
 444 }
 445
 446 # Return 0 if qemu did run (then you must check the result state/logs for actual
 447 # success), or 1 if qemu is not available.
 448 run_qemu() {
 449     if declare -F run_qemu_hook >/dev/null; then
 450         if ! run_qemu_hook "${workspace}"; then
 451             derror "check_qemu_hook() returned with EC > 0"
 452             ret=4
 453         fi
 454     fi
 455
 456     # If the test provided its own initrd, use it (e.g. TEST-24)
 457     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 458         INITRD="$TESTDIR/initrd.img"
 459     fi
 460
 461     if [ -f /etc/machine-id ]; then
 462         read -r MACHINE_ID </etc/machine-id
 463         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 464             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 465         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 466             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 467     fi
 468
 469     local CONSOLE=ttyS0
 470
 471     rm -f "$initdir"/{testok,failed,skipped}
 472     # make sure the initdir is not mounted to avoid concurrent access
 473     cleanup_initdir
 474     umount_loopback
 475
 476     if [[ ! "$KERNEL_BIN" ]]; then
 477         if get_bool "$LOOKS_LIKE_ARCH"; then
 478             KERNEL_BIN=/boot/vmlinuz-linux
 479         else
 480             [ "$ARCH" ] || ARCH=$(uname -m)
 481             case $ARCH in
 482                 ppc64*)
 483                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 484                     # uses the "standard" vmlinuz- prefix
 485                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 486                     CONSOLE=hvc0
 487                     ;;
 488                 *)
 489                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 490                     ;;
 491             esac
 492         fi
 493     fi
 494
 495     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 496     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 497     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 498     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 499     if [[ ! "$INITRD" ]]; then
 500         if [[ -e "$default_fedora_initrd" ]]; then
 501             INITRD="$default_fedora_initrd"
 502         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 503             INITRD="$default_debian_initrd"
 504         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 505             INITRD="$default_arch_initrd"
 506         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 507             INITRD="$default_suse_initrd"
 508         fi
 509     fi
 510
 511     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 512     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 513     # is not installed or there's some other error when calling it, fall back
 514     # to the original value (QEMU_SMP=1).
 515     if [[ -z "${QEMU_SMP:=}" ]]; then
 516         if ! QEMU_SMP=$(nproc); then
 517             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 518             QEMU_SMP=1
 519         fi
 520     fi
 521
 522     find_qemu_bin || return 1
 523
 524     if get_bool "${TEST_SETUP_SWTPM:-}"; then
 525         qemu_setup_swtpm_socket || return 1
 526     fi
 527
 528     # Umount initdir to avoid concurrent access to the filesystem
 529     _umount_dir "$initdir"
 530
 531     local kernel_params=()
 532     local qemu_options=()
 533     local qemu_cmd=("$QEMU_BIN")
 534
 535     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 536         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 537     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 538         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 539     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 540         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 541     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 542         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 543         exit 1
 544     fi
 545
 546     if get_bool "$LOOKS_LIKE_SUSE"; then
 547         kernel_params+=("rd.hostonly=0")
 548     fi
 549
 550     # Debian/Ubuntu's initramfs tries to check if it can resume from hibernation
 551     # and wastes a minute or so probing disks, skip that as it's not useful here
 552     kernel_params+=(
 553         "root=LABEL=systemd_boot"
 554         "rw"
 555         "raid=noautodetect"
 556         "rd.luks=0"
 557         "loglevel=2"
 558         "init=$PATH_TO_INIT"
 559         "console=$CONSOLE"
 560         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 561         "systemd.unit=testsuite.target"
 562         "systemd.wants=testsuite-$1.service"
 563         "noresume"
 564         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 565         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 566     )
 567
 568     if ! get_bool "$INTERACTIVE_DEBUG"; then
 569         kernel_params+=(
 570             "oops=panic"
 571             "panic=1"
 572             "systemd.wants=end.service"
 573         )
 574     fi
 575
 576     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 577     qemu_options+=(
 578         -smp "$QEMU_SMP"
 579         -net none
 580         -m "${QEMU_MEM:-768M}"
 581         -nographic
 582         -kernel "$KERNEL_BIN"
 583         -drive "format=raw,cache=unsafe,file=$image"
 584         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 585     )
 586
 587     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 588         local user_qemu_options
 589         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 590         qemu_options+=("${user_qemu_options[@]}")
 591     fi
 592     qemu_options+=(${QEMU_OPTIONS_ARRAY:+"${QEMU_OPTIONS_ARRAY[@]}"})
 593
 594     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 595         local user_kernel_append
 596         readarray user_kernel_append <<< "$KERNEL_APPEND"
 597         kernel_params+=("${user_kernel_append[@]}")
 598     fi
 599
 600     if [[ -n "$INITRD" ]]; then
 601         if [[ -n "$INITRD_EXTRA" ]]; then
 602             # An addition initrd has been specified, let's combine it with the main one.
 603             local t="$WORKDIR"/initrd.combined."$RANDOM"
 604
 605             # First, show contents of additional initrd
 606             echo "Additional initrd contents:"
 607             cpio -tv < "$INITRD_EXTRA"
 608
 609             # Copy the main initrd
 610             zstd -d -c -f "$INITRD" > "$t"
 611             add_at_exit_handler "rm $t"
 612             # Kernel requires this to be padded to multiple of 4 bytes with zeroes
 613             pad4_file "$t"
 614
 615             # Copy the additional initrd
 616             cat "$INITRD_EXTRA" >> "$t"
 617             pad4_file "$t"
 618
 619             qemu_options+=(-initrd "$t")
 620         else
 621             qemu_options+=(-initrd "$INITRD")
 622         fi
 623     fi
 624
 625     # Let's use KVM if possible
 626     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 627         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 628     fi
 629
 630     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 631         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 632     fi
 633
 634     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]}" |& tee "${TESTDIR:?}/console.log")
 635     rc=$?
 636     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 637         derror "Test timed out after ${QEMU_TIMEOUT}s"
 638         TIMED_OUT=1
 639     else
 640         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 641     fi
 642     return 0
 643 }
 644
 645 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 646 # success), or 1 if nspawn is not available.
 647 run_nspawn() {
 648     [[ -d /run/systemd/system ]] || return 1
 649     rm -f "${initdir:?}"/{testok,failed,skipped}
 650
 651     local nspawn_cmd=()
 652     local nspawn_options=(
 653         "--register=no"
 654         "--kill-signal=SIGKILL"
 655         "--directory=${1:?}"
 656         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 657         "--machine=TEST-$TESTID"
 658     )
 659     local kernel_params=(
 660         "$PATH_TO_INIT"
 661         "systemd.unit=testsuite.target"
 662         "systemd.wants=testsuite-$2.service"
 663         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 664         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 665     )
 666
 667     if get_bool "$INTERACTIVE_DEBUG"; then
 668         nspawn_options+=("--console=interactive")
 669     else
 670         kernel_params+=("systemd.wants=end.service")
 671     fi
 672
 673     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 674         local user_nspawn_arguments
 675         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 676         nspawn_options+=("${user_nspawn_arguments[@]}")
 677     fi
 678
 679     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 680         local user_kernel_append
 681         readarray user_kernel_append <<< "$KERNEL_APPEND"
 682         kernel_params+=("${user_kernel_append[@]}")
 683     fi
 684
 685     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 686         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 687         exit
 688     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 689         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 690     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 691         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 692     else
 693         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 694         exit 1
 695     fi
 696
 697     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 698         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 699     else
 700         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 701     fi
 702
 703     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}" |& tee "${TESTDIR:?}/console.log")
 704     rc=$?
 705     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 706         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 707         TIMED_OUT=1
 708     else
 709         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 710     fi
 711     return 0
 712 }
 713
 714 # Build two very minimal root images, with two units, one is the same and one is different across them
 715 install_verity_minimal() {
 716     dinfo "Set up a set of minimal images for verity verification"
 717     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 718         return
 719     fi
 720     if ! command -v mksquashfs >/dev/null 2>&1; then
 721         dfatal "mksquashfs not found"
 722         exit 1
 723     fi
 724     if ! command -v veritysetup >/dev/null 2>&1; then
 725         dfatal "veritysetup not found"
 726         exit 1
 727     fi
 728     # Local modifications of some global variables is intentional in this
 729     # subshell (SC2030)
 730     # shellcheck disable=SC2030
 731     (
 732         BASICTOOLS=(
 733             bash
 734             cat
 735             grep
 736             mount
 737             sleep
 738             touch
 739         )
 740         oldinitdir="$initdir"
 741         rm -rfv "$TESTDIR/minimal"
 742         export initdir="$TESTDIR/minimal"
 743         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 744         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 745         setup_basic_dirs
 746         install_basic_tools
 747         install_ld_so_conf
 748         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 749         # issue, thus falsely triggering SC2030 in this case
 750         # See: koalaman/shellcheck#1409
 751         if [[ -v ASAN_RT_PATH ]]; then
 752             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 753             # into the verity images to get rid of the annoying errors about
 754             # missing $LD_PRELOAD libraries.
 755             inst_libs "$ASAN_RT_PATH"
 756             inst_library "$ASAN_RT_PATH"
 757         fi
 758         cp "$os_release" "$initdir/usr/lib/os-release"
 759         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 760         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 761         touch "$initdir/opt/some_file"
 762         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 763         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 764         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 765 [Service]
 766 ExecStartPre=cat /usr/lib/os-release
 767 ExecStart=sleep 120
 768 EOF
 769         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 770
 771         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 772         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 773             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 774
 775         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 776         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 777         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 778
 779         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 780         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 781             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 782
 783         # Rolling distros like Arch do not set VERSION_ID
 784         local version_id=""
 785         if grep -q "^VERSION_ID=" "$os_release"; then
 786             version_id="$(grep "^VERSION_ID=" "$os_release")"
 787         fi
 788
 789         export initdir="$TESTDIR/app0"
 790         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 791         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 792         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 793         ( echo "${version_id}"
 794           echo "SYSEXT_IMAGE_ID=app" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 795         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 796 [Service]
 797 Type=oneshot
 798 RemainAfterExit=yes
 799 ExecStart=/opt/script0.sh
 800 TemporaryFileSystem=/var/lib
 801 StateDirectory=app0
 802 RuntimeDirectory=app0
 803 EOF
 804         cat >"$initdir/opt/script0.sh" <<EOF
 805 #!/bin/bash
 806 set -e
 807 test -e /usr/lib/os-release
 808 echo bar >\${STATE_DIRECTORY}/foo
 809 cat /usr/lib/extension-release.d/extension-release.app0
 810 EOF
 811         chmod +x "$initdir/opt/script0.sh"
 812         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 813         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 814
 815         export initdir="$TESTDIR/app1"
 816         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 817         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 818         ( echo "${version_id}"
 819           echo "SYSEXT_SCOPE=portable"
 820           echo "SYSEXT_IMAGE_ID=app"
 821           echo "SYSEXT_IMAGE_VERSION=1"
 822           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 823         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 824         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 825 [Service]
 826 Type=oneshot
 827 RemainAfterExit=yes
 828 ExecStart=/opt/script1.sh
 829 StateDirectory=app1
 830 RuntimeDirectory=app1
 831 EOF
 832         cat >"$initdir/opt/script1.sh" <<EOF
 833 #!/bin/bash
 834 set -e
 835 test -e /usr/lib/os-release
 836 echo baz >\${STATE_DIRECTORY}/foo
 837 cat /usr/lib/extension-release.d/extension-release.app2
 838 EOF
 839         chmod +x "$initdir/opt/script1.sh"
 840         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 841         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 842
 843         export initdir="$TESTDIR/app-nodistro"
 844         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 845         ( echo "ID=_any"
 846           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 847         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 848         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 849
 850         export initdir="$TESTDIR/service-scoped-test"
 851         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system"
 852         ( echo "ID=_any"
 853           echo "ARCHITECTURE=_any" ) >"$initdir/etc/extension-release.d/extension-release.service-scoped-test"
 854         echo MARKER_CONFEXT_123 >"$initdir/etc/systemd/system/some_file"
 855         mksquashfs "$initdir" "$oldinitdir/etc/service-scoped-test.raw" -noappend
 856
 857         # We need to create a dedicated sysext image to test the reload mechanism. If we share an image to install the
 858         # 'foo.service' it will be loaded from another test run, which will impact the targeted test.
 859         export initdir="$TESTDIR/app-reload"
 860         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 861         ( echo "ID=_any"
 862           echo "ARCHITECTURE=_any"
 863           echo "EXTENSION_RELOAD_MANAGER=1" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-reload"
 864         mkdir -p "$initdir/usr/lib/systemd/system/multi-user.target.d"
 865         cat >"${initdir}/usr/lib/systemd/system/foo.service" <<EOF
 866 [Service]
 867 Type=oneshot
 868 RemainAfterExit=yes
 869 ExecStart=echo foo
 870
 871 [Install]
 872 WantedBy=multi-user.target
 873 EOF
 874         { echo "[Unit]"; echo "Upholds=foo.service"; } > "$initdir/usr/lib/systemd/system/multi-user.target.d/10-foo-service.conf"
 875         mksquashfs "$initdir" "$oldinitdir/usr/share/app-reload.raw" -noappend
 876     )
 877 }
 878
 879 setup_basic_environment() {
 880     # create the basic filesystem layout
 881     setup_basic_dirs
 882
 883     install_systemd
 884     install_missing_libraries
 885     install_config_files
 886     install_zoneinfo
 887     create_rc_local
 888     install_basic_tools
 889     install_libnss
 890     install_pam
 891     install_dbus
 892     install_fonts
 893     install_locales
 894     install_keymaps
 895     install_x11_keymaps
 896     install_terminfo
 897     install_execs
 898     install_fs_tools
 899     install_modules
 900     install_plymouth
 901     install_haveged
 902     install_debug_tools
 903     install_ld_so_conf
 904     install_testuser
 905     has_user_dbus_socket && install_user_dbus
 906     setup_selinux
 907     install_depmod_files
 908     generate_module_dependencies
 909     if get_bool "$IS_BUILT_WITH_ASAN"; then
 910         create_asan_wrapper
 911     fi
 912     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 913         install_verity_minimal
 914     fi
 915 }
 916
 917 setup_selinux() {
 918     dinfo "Setup SELinux"
 919     # don't forget KERNEL_APPEND='... selinux=1 ...'
 920     if ! get_bool "$SETUP_SELINUX"; then
 921         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 922         return 0
 923     fi
 924
 925     local conf_dir=/etc/selinux
 926     local fixfiles_tools=(awk bash cat chcon expr egrep find grep head secon setfiles rm sort uname uniq)
 927
 928     # Make sure the following statement can't expand to "/" to prevent
 929     # a potential where-are-my-backups situation
 930     rm -rf "${initdir:?}/$conf_dir"
 931     if ! cp -ar "$conf_dir" "$initdir/$conf_dir"; then
 932         dfatal "Failed to copy $conf_dir"
 933         exit 1
 934     fi
 935
 936     # We use a custom autorelabel service instead of the SELinux provided set
 937     # of units & a generator, since the generator overrides the default target
 938     # to the SELinux one when it detects /.autorelabel. However, we use
 939     # systemd.unit= on the kernel command cmdline which always takes precedence,
 940     # rendering all SELinux efforts useless. Also, pulling in selinux-autorelabel.service
 941     # explicitly doesn't work either, as it doesn't check for the presence of /.autorelabel
 942     # and does the relabeling unconditionally which always ends with a reboot, so
 943     # we end up in a reboot loop (and it also spews quite a lot of errors as it
 944     # wants /etc/fstab and dracut-initramfs-restore).
 945
 946     touch "$initdir/.autorelabel"
 947     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 948     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 949
 950     image_install "${fixfiles_tools[@]}"
 951     image_install fixfiles sestatus
 952 }
 953
 954 install_valgrind() {
 955     if ! type -p valgrind; then
 956         dfatal "Failed to install valgrind"
 957         exit 1
 958     fi
 959
 960     local valgrind_bins valgrind_libs valgrind_supp
 961
 962     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null |
 963                                        sed -r -n 's/execve\("([^"]*)".*/\1/p')
 964     image_install "${valgrind_bins[@]}"
 965
 966     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null |
 967                                        sed -r -n 's|.*calling init: (/.*vgpreload_.*)|\1|p')
 968     image_install "${valgrind_libs[@]}"
 969
 970     readarray -t valgrind_supp < <(strace -e open valgrind /bin/true 2>&1 >/dev/null |
 971                                        sed -r -n 's,open\("([^"]*(/debug[^"]*|\.supp))".*= [0-9].*,\1,p')
 972     image_install "${valgrind_supp[@]}"
 973 }
 974
 975 create_valgrind_wrapper() {
 976     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 977     ddebug "Create $valgrind_wrapper"
 978     cat >"$valgrind_wrapper" <<EOF
 979 #!/usr/bin/env bash
 980
 981 mount -t proc proc /proc
 982 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 983 EOF
 984     chmod 0755 "$valgrind_wrapper"
 985 }
 986
 987 create_asan_wrapper() {
 988     local asan_wrapper default_asan_options default_ubsan_options default_environment manager_environment
 989
 990     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 991
 992     default_asan_options="${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}"
 993     default_ubsan_options="${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}"
 994
 995     if [[ "$ASAN_COMPILER" == "clang" ]]; then
 996         # clang: install llvm-symbolizer to generate useful reports
 997         # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
 998         image_install "llvm-symbolizer"
 999
1000         # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
1001         # unnecessary for gcc & libasan, however, for clang this is crucial, as its
1002         # runtime ASan DSO is in a non-standard (library) path.
1003         mkdir -p "${initdir:?}/etc/ld.so.conf.d/"
1004         echo "${ASAN_RT_PATH%/*}" >"${initdir:?}/etc/ld.so.conf.d/asan-path-override.conf"
1005         ldconfig -r "$initdir"
1006     fi
1007
1008     # Create a simple environment file which can be included by systemd services
1009     # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
1010     cat >"${initdir:?}/usr/lib/systemd/systemd-asan-env" <<EOF
1011 LD_PRELOAD=$ASAN_RT_PATH
1012 ASAN_OPTIONS=$default_asan_options
1013 LSAN_OPTIONS=detect_leaks=0
1014 UBSAN_OPTIONS=$default_ubsan_options
1015 EOF
1016
1017     default_environment=(
1018         "ASAN_OPTIONS='$default_asan_options'"
1019         "UBSAN_OPTIONS='$default_ubsan_options'"
1020         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1021     )
1022     manager_environment=(
1023         "ASAN_OPTIONS='$default_asan_options:log_path=/systemd-pid1.asan.log:log_to_syslog=1'"
1024         "UBSAN_OPTIONS='$default_ubsan_options:log_path=/systemd-pid1.ubsan.log:log_to_syslog=1'"
1025         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1026     )
1027
1028     mkdir -p "${initdir:?}/etc/systemd/system.conf.d/"
1029     cat >"${initdir:?}/etc/systemd/system.conf.d/asan.conf" <<EOF
1030 [Manager]
1031 DefaultEnvironment=${default_environment[*]}
1032 ManagerEnvironment=${manager_environment[*]}
1033 DefaultTimeoutStartSec=180s
1034 DefaultStandardOutput=journal+console
1035 EOF
1036
1037     # ASAN and syscall filters aren't compatible with each other.
1038     find "${initdir:?}" -name '*.service' -type f -print0 | xargs -0 sed -i 's/^\(MemoryDeny\|SystemCall\)/#\1/'
1039
1040     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journald.service.d/"
1041     cat >"${initdir:?}/etc/systemd/system/systemd-journald.service.d/asan-env.conf" <<EOF
1042 [Service]
1043 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
1044 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
1045 Environment=ASAN_OPTIONS=$default_asan_options:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=$default_ubsan_options:log_path=/systemd-journald.ubsan.log
1046
1047 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
1048 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
1049 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
1050 StandardOutput=file:/systemd-journald.out
1051 EOF
1052
1053     # 90s isn't enough for some services to finish when literally everything is run
1054     # under ASan+UBSan in containers, which, in turn, are run in VMs.
1055     # Let's limit which environments such services should be executed in.
1056     mkdir -p "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/"
1057     cat >"${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/asan.conf" <<EOF
1058 [Unit]
1059 ConditionVirtualization=container
1060
1061 [Service]
1062 TimeoutSec=240s
1063 EOF
1064
1065     # Let's override another hard-coded timeout that kicks in too early
1066     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/"
1067     cat >"${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/asan.conf" <<EOF
1068 [Service]
1069 TimeoutSec=180s
1070 EOF
1071
1072     asan_wrapper="${initdir:?}/${PATH_TO_INIT:?}"
1073     # Sanity check to make sure we don't overwrite something we shouldn't.
1074     [[ "$asan_wrapper" =~ systemd-under-asan$ ]]
1075
1076     cat >"$asan_wrapper" <<EOF
1077 #!/usr/bin/env bash
1078 set -eux
1079
1080 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
1081 export ${manager_environment[@]}
1082 [[ -n "\$ASAN_OPTIONS" && -n "\$UBSAN_OPTIONS" ]]
1083
1084 exec "$ROOTLIBDIR/systemd" "\$@"
1085 EOF
1086     chmod 0755 "$asan_wrapper"
1087 }
1088
1089 create_strace_wrapper() {
1090     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
1091     ddebug "Create $strace_wrapper"
1092     cat >"$strace_wrapper" <<EOF
1093 #!/usr/bin/env bash
1094
1095 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
1096 EOF
1097     chmod 0755 "$strace_wrapper"
1098 }
1099
1100 install_fs_tools() {
1101     dinfo "Install fsck"
1102     image_install /sbin/fsck*
1103     image_install -o /bin/fsck*
1104
1105     # fskc.reiserfs calls reiserfsck. so, install it
1106     image_install -o reiserfsck
1107
1108     # we use mkfs in system-repart tests
1109     image_install /sbin/mkfs.ext4
1110     image_install /sbin/mkfs.vfat
1111 }
1112
1113 install_modules() {
1114     dinfo "Install modules"
1115
1116     instmods bridge dummy ipvlan macvlan vfat veth
1117     instmods loop =block
1118     instmods nls_ascii =nls
1119     instmods overlay =overlayfs
1120     instmods scsi_debug
1121
1122     if get_bool "$LOOKS_LIKE_SUSE"; then
1123         instmods ext4 af_packet
1124     fi
1125 }
1126
1127 install_dmevent() {
1128     instmods dm_crypt =crypto
1129     inst_binary dmeventd
1130     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
1131     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1132         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
1133         # and since buster/bionic 95-dm-notify.rules
1134         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
1135         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
1136     else
1137         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
1138     fi
1139     if get_bool "$LOOKS_LIKE_SUSE"; then
1140         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
1141     fi
1142 }
1143
1144 install_multipath() {
1145     instmods "=md" multipath
1146     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
1147     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
1148     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1149         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
1150     else
1151         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
1152     fi
1153     mkdir -p "${initdir:?}/etc/multipath"
1154
1155     local file
1156     while read -r file; do
1157         # Install libraries required by the given library
1158         inst_libs "$file"
1159         # Install the library itself and create necessary symlinks
1160         inst_library "$file"
1161     done < <(find /lib*/multipath -type f)
1162 }
1163
1164 install_lvm() {
1165     image_install lvm
1166     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1167     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1168     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1169     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1170         inst_rules 56-lvm.rules 69-lvm-metad.rules
1171     else
1172         # Support the new udev autoactivation introduced in lvm 2.03.14
1173         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1174         # Static autoactivation (via lvm2-activation-generator) was dropped
1175         # in lvm 2.03.15
1176         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1177         if [[ -f /lib/udev/rules.d/69-dm-lvm.rules ]]; then
1178             inst_rules 11-dm-lvm.rules 69-dm-lvm.rules
1179         else
1180             image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1181             image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1182             inst_rules 11-dm-lvm.rules 69-dm-lvm-metad.rules
1183         fi
1184     fi
1185     mkdir -p "${initdir:?}/etc/lvm"
1186 }
1187
1188 install_btrfs() {
1189     instmods btrfs
1190     # Not all utilities provided by btrfs-progs are listed here; extend the list
1191     # if necessary
1192     image_install btrfs btrfstune mkfs.btrfs
1193     inst_rules 64-btrfs-dm.rules
1194 }
1195
1196 install_iscsi() {
1197     # Install both client and server side stuff by default
1198     local inst="${1:-}"
1199     local file
1200
1201     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1202     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1203     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1204         image_install iscsi-iname iscsiadm iscsid iscsistart
1205         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1206         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1207         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1208         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1209         mkdir -p "${initdir:?}/etc/iscsi"
1210         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1211         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1212         # a one-time service instead of distro package's post-install scripts.
1213         # However, some distros still use this approach even after this patch,
1214         # so prefer the already existing initiatorname.iscsi file if it exists.
1215         #
1216         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1217         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1218             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1219             if get_bool "$IS_BUILT_WITH_ASAN"; then
1220                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1221                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1222                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1223                 # (among others) pre-loads ASan's DSO.
1224                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1225                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1226             fi
1227         else
1228             inst_simple "/etc/iscsi/initiatorname.iscsi"
1229         fi
1230     fi
1231
1232     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1233     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1234     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1235         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1236         image_install -o /etc/sysconfig/tgtd
1237         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1238         mkdir -p "${initdir:?}/etc/tgt"
1239         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1240         # Install perl modules required by tgt-admin
1241         #
1242         # Forgive me father for I have sinned. The monstrosity below appends
1243         # a perl snippet to the `tgt-admin` perl script on the fly, which
1244         # dumps a list of files (perl modules) required by `tgt-admin` at
1245         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1246         # passed to `inst_simple` which installs the necessary files into the image
1247         #
1248         # shellcheck disable=SC2016
1249         while read -r file; do
1250             inst_simple "$file"
1251         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1252     fi
1253 }
1254
1255 install_mdadm() {
1256     local unit
1257     local mdadm_units=(
1258         system/mdadm-grow-continue@.service
1259         system/mdadm-last-resort@.service
1260         system/mdadm-last-resort@.timer
1261         system/mdmon@.service
1262         system/mdmonitor-oneshot.service
1263         system/mdmonitor-oneshot.timer
1264         system/mdmonitor.service
1265         system-shutdown/mdadm.shutdown
1266     )
1267
1268     image_install mdadm mdmon
1269     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1270     # Fedora/CentOS/RHEL ships this rule file
1271     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1272
1273     for unit in "${mdadm_units[@]}"; do
1274         image_install "${ROOTLIBDIR:?}/$unit"
1275     done
1276 }
1277
1278 install_compiled_systemd() {
1279     dinfo "Install compiled systemd"
1280
1281     local ninja_bin
1282     ninja_bin="$(type -P ninja || type -P ninja-build)"
1283     if [[ -z "$ninja_bin" ]]; then
1284         dfatal "ninja was not found"
1285         exit 1
1286     fi
1287     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1288
1289     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1290     # find them in the same directory as the runtime data counts
1291     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1292         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1293         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1294         # Set effective & default ACLs for the build dir so unprivileged
1295         # processes can write gcda files with coverage stats
1296         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1297     fi
1298 }
1299
1300 install_debian_systemd() {
1301     dinfo "Install debian systemd"
1302
1303     local files
1304
1305     while read -r deb; do
1306         files="$(dpkg-query -L "$deb" 2>/dev/null)" || continue
1307         ddebug "Install debian files from package $deb"
1308         for file in $files; do
1309             [ -e "$file" ] || continue
1310             [ ! -L "$file" ] && [ -d "$file" ] && continue
1311             inst "$file"
1312         done
1313     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1314 }
1315
1316 install_suse_systemd() {
1317     local pkgs
1318
1319     dinfo "Install SUSE systemd"
1320
1321     pkgs=(
1322         systemd
1323         systemd-container
1324         systemd-coredump
1325         systemd-experimental
1326         systemd-homed
1327         systemd-journal-remote
1328         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1329         # systemd-network sub-package on openSUSE, has its own testsuite.
1330         systemd-network
1331         systemd-portable
1332         udev
1333     )
1334
1335     for p in "${pkgs[@]}"; do
1336         rpm -q "$p" &>/dev/null || continue
1337
1338         ddebug "Install files from package $p"
1339         while read -r f; do
1340             [ -e "$f" ] || continue
1341             [ ! -L "$f" ] && [ -d "$f" ] && continue
1342             inst "$f"
1343         done < <(rpm -ql "$p")
1344     done
1345
1346     dinfo "Install the data needed by the tests at runtime"
1347     inst_recursive "${SOURCE_DIR}/testdata"
1348     inst_recursive "${SOURCE_DIR}/unit-tests/manual"
1349
1350     # On openSUSE, this directory is not created at package install, at least
1351     # for now.
1352     mkdir -p "$initdir/var/log/journal/remote"
1353 }
1354
1355 install_distro_systemd() {
1356     dinfo "Install distro systemd"
1357
1358     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1359         install_debian_systemd
1360     elif get_bool "$LOOKS_LIKE_SUSE"; then
1361         install_suse_systemd
1362     else
1363         dfatal "NO_BUILD not supported for this distro"
1364         exit 1
1365     fi
1366 }
1367
1368 install_systemd() {
1369     dinfo "Install systemd"
1370     if get_bool "$NO_BUILD"; then
1371         install_distro_systemd
1372     else
1373         install_compiled_systemd
1374     fi
1375
1376     # Remove unneeded documentation
1377     rm -fr "${initdir:?}"/usr/share/{man,doc}
1378
1379     # Enable debug logging in PID1
1380     mkdir -p "$initdir/etc/systemd/system.conf.d/"
1381     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/system.conf.d/10-log-level.conf"
1382     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1383         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf.d/99-log-level.conf"
1384     fi
1385     # Enable debug logging for user instances as well
1386     mkdir -p "$initdir/etc/systemd/user.conf.d/"
1387     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/user.conf.d/10-log-level.conf"
1388     # Store coredumps in journal
1389     mkdir -p "$initdir/etc/systemd/coredump.conf.d/"
1390     echo -ne "[Coredump]\nStorage=journal\n" >"$initdir/etc/systemd/coredump.conf.d/10-storage-journal.conf"
1391     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1392     mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
1393     echo -ne "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/99-SYSTEMD_UNIT_PATH.conf"
1394
1395     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1396     # images, since it prevents gcov to write the coverage reports (*.gcda
1397     # files)
1398     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1399         mkdir -p "$initdir/etc/systemd/system/service.d/"
1400         echo -ne "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1401         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image to make the coverage work with
1402         # units using DynamicUser=yes. Do this only for services with test- prefix and a couple of
1403         # known-to-use DynamicUser=yes services, as setting this system-wide has many undesirable
1404         # side-effects, as it creates its own namespace.
1405         for service in test- systemd-journal-{gatewayd,upload}; do
1406             mkdir -p "$initdir/etc/systemd/system/$service.service.d/"
1407             echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/$service.service.d/99-gcov-rwpaths-override.conf"
1408         done
1409         # Ditto, but for the user daemon
1410         mkdir -p "$initdir/etc/systemd/user/test-.service.d/"
1411         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf"
1412         # Bind the $BUILD_DIR into nspawn containers that are executed using
1413         # machinectl. Unfortunately, the .nspawn files don't support drop-ins
1414         # so we have to inject the bind mount directly into
1415         # the systemd-nspawn@.service unit.
1416         cp "$initdir/usr/lib/systemd/system/systemd-nspawn@.service" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1417         sed -ri "s/^ExecStart=.+$/& --bind=${BUILD_DIR//\//\\\/}/" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1418         # Pass the $BUILD_DIR as $COVERAGE_BUILD_DIR env variable to the system
1419         # manager, similarly to what we do with $ASAN_RT_PATH during sanitized
1420         # builds
1421         mkdir -p "$initdir/etc/systemd/system.conf.d/"
1422         echo -ne "[Manager]\nDefaultEnvironment=COVERAGE_BUILD_DIR=$BUILD_DIR\n" >"$initdir/etc/systemd/system.conf.d/99-COVERAGE_BUILD_DIR.conf"
1423     fi
1424
1425     # If we're built with -Dportabled=false, tests with systemd-analyze
1426     # --profile will fail. Since we need just the profile (text) files, let's
1427     # copy them into the image if they don't exist there.
1428     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1429     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1430         dinfo "Couldn't find portable profiles in the test image"
1431         dinfo "Copying them directly from the source tree"
1432         mkdir -p "$portable_dir"
1433         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1434     fi
1435 }
1436
1437 get_ldpath() {
1438     local rpath
1439     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1440
1441     if [ -z "$rpath" ] ; then
1442         echo "$BUILD_DIR"
1443     else
1444         echo "$rpath"
1445     fi
1446 }
1447
1448 install_missing_libraries() {
1449     dinfo "Install missing libraries"
1450     # install possible missing libraries
1451     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/unit-tests/{,manual/,unsafe/}}*; do
1452         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1453     done
1454
1455     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1456     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1457     # if not present
1458     # See: https://github.com/systemd/systemd/pull/23858
1459     while read -r libgcc_s; do
1460         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1461     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1462
1463     local lib path
1464     # A number of dependencies is now optional via dlopen, so the install
1465     # script will not pick them up, since it looks at linkage.
1466     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1; do
1467         ddebug "Searching for $lib via pkg-config"
1468         if pkg-config --exists "$lib"; then
1469                 path="$(pkg-config --variable=libdir "$lib")"
1470                 if [ -z "${path}" ]; then
1471                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1472                     continue
1473                 fi
1474
1475                 if ! [[ ${lib} =~ ^lib ]]; then
1476                         lib="lib${lib}"
1477                 fi
1478                 # p11-kit-1's .so doesn't have the API level in the name
1479                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1480                         lib="libp11-kit"
1481                 fi
1482                 # Some pkg-config files are broken and give out the wrong paths
1483                 # (eg: libcryptsetup), so just ignore them
1484                 inst_libs "${path}/${lib}.so" || true
1485                 inst_library "${path}/${lib}.so" || true
1486
1487                 if [[ "$lib" == "libxkbcommon" ]]; then
1488                     install_x11_keymaps full
1489                 fi
1490         else
1491             ddebug "$lib.pc not found, skipping"
1492             continue
1493         fi
1494     done
1495
1496     # Install extra openssl 3 stuff
1497     path="$(pkg-config --variable=libdir libcrypto)"
1498     inst_simple "${path}/ossl-modules/legacy.so" || true
1499     inst_simple "${path}/ossl-modules/fips.so" || true
1500     inst_simple "${path}/engines-3/afalg.so" || true
1501     inst_simple "${path}/engines-3/capi.so" || true
1502     inst_simple "${path}/engines-3/loader_attic.so" || true
1503     inst_simple "${path}/engines-3/padlock.so" || true
1504
1505     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1506     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1507     # shellcheck disable=SC2046
1508     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1509 }
1510
1511 cleanup_loopdev() {
1512     if [ -n "${LOOPDEV:=}" ]; then
1513         ddebug "losetup -d $LOOPDEV"
1514         losetup -d "${LOOPDEV}"
1515         unset LOOPDEV
1516     fi
1517 }
1518
1519 add_at_exit_handler cleanup_loopdev
1520
1521 create_empty_image() {
1522     if [[ -z "${IMAGE_NAME:=}" ]]; then
1523         echo "create_empty_image: \$IMAGE_NAME not set"
1524         exit 1
1525     fi
1526
1527     # Partition sizes are in MiBs
1528     local root_size=768
1529     local data_size=100
1530     if ! get_bool "$NO_BUILD"; then
1531         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1532             root_size=$((root_size + 200))
1533         fi
1534         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1535             root_size=$((root_size + 200))
1536         fi
1537         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1538             root_size=$((root_size + 250))
1539         fi
1540         if get_bool "$IS_BUILT_WITH_ASAN"; then
1541             root_size=$((root_size * 2))
1542         fi
1543     fi
1544
1545     if [[ "${IMAGE_ADDITIONAL_ROOT_SIZE:-0}" -gt 0 ]]; then
1546         root_size=$((root_size + IMAGE_ADDITIONAL_ROOT_SIZE))
1547     fi
1548     if [[ "${IMAGE_ADDITIONAL_DATA_SIZE:-0}" -gt 0 ]]; then
1549         data_size=$((data_size + IMAGE_ADDITIONAL_DATA_SIZE))
1550     fi
1551
1552     echo "Setting up ${IMAGE_PUBLIC:?} (${root_size} MB)"
1553     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1554
1555     # Create the blank file to use as a root filesystem
1556     truncate -s "${root_size}M" "$IMAGE_PUBLIC"
1557
1558     LOOPDEV="$(losetup --show -P -f "$IMAGE_PUBLIC")"
1559     [[ -b "$LOOPDEV" ]] || return 1
1560     # Create two partitions - a root one and a data one (utilized by some tests)
1561     sfdisk "$LOOPDEV" <<EOF
1562 label: gpt
1563 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=root size=$((root_size - data_size))M bootable
1564 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1565 EOF
1566
1567     udevadm settle
1568
1569     local label=(-L systemd_boot)
1570     # mkfs.reiserfs doesn't know -L. so, use --label instead
1571     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1572     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p1" -q; then
1573         dfatal "Failed to mkfs -t ${FSTYPE}"
1574         exit 1
1575     fi
1576 }
1577
1578 mount_initdir() {
1579     if [ -z "${LOOPDEV:=}" ]; then
1580         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1581         LOOPDEV="$(losetup --show -P -f "$image")"
1582         [ -b "$LOOPDEV" ] || return 1
1583
1584         udevadm settle
1585     fi
1586
1587     if ! mountpoint -q "${initdir:?}"; then
1588         mkdir -p "$initdir"
1589         mount "${LOOPDEV}p1" "$initdir"
1590         TEST_SETUP_CLEANUP_ROOTDIR=1
1591     fi
1592 }
1593
1594 cleanup_initdir() {
1595     # only umount if create_empty_image_rootdir() was called to mount it
1596     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1597         _umount_dir "${initdir:?}"
1598     fi
1599 }
1600
1601 umount_loopback() {
1602     # unmount the loopback device from all places. Otherwise we risk file
1603     # system corruption.
1604     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1605         ddebug "Unmounting all uses of $device"
1606         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1607     done
1608 }
1609
1610 create_empty_image_rootdir() {
1611     create_empty_image
1612     mount_initdir
1613 }
1614
1615 check_asan_reports() {
1616     local ret=0
1617     local root="${1:?}"
1618     local log report
1619
1620     if get_bool "$IS_BUILT_WITH_ASAN"; then
1621         ls -l "$root"
1622         if [[ -e "$root/systemd.asan.log.1" ]]; then
1623             cat "$root/systemd.asan.log.1"
1624             ret=$((ret+1))
1625         fi
1626
1627         for log in pid1 journald; do
1628             report="$(find "$root" -name "systemd-$log.*san.log*" -exec cat {} \;)"
1629             if [[ -n "$report" ]]; then
1630                 printf "%s\n" "$report"
1631                 # shellcheck disable=SC2015
1632                 [[ "$log" == journald ]] && cat "$root/systemd-journald.out" || :
1633                 ret=$((ret+1))
1634             fi
1635         done
1636
1637         # May 08 13:23:31 H testleak[2907148]: SUMMARY: AddressSanitizer: 4 byte(s) leaked in 1 allocation(s).
1638         pids="$(
1639             "$JOURNALCTL" -D "$root/var/log/journal" --grep 'SUMMARY: .*Sanitizer:' |
1640                    grep -v -E 'dbus-daemon|dbus-broker-launch' |
1641                    sed -r -n 's/.* .+\[([0-9]+)\]: SUMMARY:.*/\1/p'
1642         )"
1643
1644         if [[ -n "$pids" ]]; then
1645             ret=$((ret+1))
1646             for pid in $pids; do
1647                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1648             done
1649         fi
1650     fi
1651
1652     return $ret
1653 }
1654
1655 check_coverage_reports() {
1656     local root="${1:?}"
1657
1658     if get_bool "$NO_BUILD"; then
1659         return 0
1660     fi
1661     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1662         return 0
1663     fi
1664
1665     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1666         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1667     else
1668         dest="${TESTDIR:?}/coverage-info"
1669     fi
1670
1671     if [[ ! -e "${TESTDIR:?}/coverage-base" ]]; then
1672         # This shouldn't happen, as the report is generated during the setup
1673         # phase (test_setup()).
1674         derror "Missing base coverage report"
1675         return 1
1676     fi
1677
1678     # Create a coverage report that will later be uploaded. Remove info about system
1679     # libraries/headers and generated files, as we don't really care about them.
1680     lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1681     if [[ -f "$dest" ]]; then
1682         # If the destination report file already exists, don't overwrite it, but
1683         # merge it with the already present one - this usually happens when
1684         # running both "parts" of a test in one run (the qemu and the nspawn part).
1685         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1686     else
1687         # If there's no prior coverage report, merge the new one with the base
1688         # report we did during the setup phase (see test_setup()).
1689         lcov --add-tracefile "${TESTDIR:?}/coverage-base" --add-tracefile "${dest}.new" -o "${dest}"
1690     fi
1691     lcov --remove "$dest" -o "$dest" '/usr/include/*' '/usr/lib/*' "${BUILD_DIR:?}/*"
1692     rm -f "${dest}.new"
1693
1694     # If the test logs contain lines like:
1695     #
1696     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1697     #
1698     # it means we're possibly missing some coverage since gcov can't write the stats,
1699     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1700     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1701     # `setfacl` stuff in install_compiled_systemd().
1702     #
1703     # Also, a note: some tests, like TEST-46, overmount /home with tmpfs, which
1704     # means if your build dir is under /home/your-user (which is usually the
1705     # case) you might get bogus errors and missing coverage.
1706     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1707        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1708         derror "Detected possibly missing coverage, check the journal"
1709         return 1
1710     fi
1711
1712     return 0
1713 }
1714
1715 save_journal() {
1716     # Default to always saving journal
1717     local save="yes"
1718
1719     if [ "${TEST_SAVE_JOURNAL}" = "no" ]; then
1720         save="no"
1721     elif [ "${TEST_SAVE_JOURNAL}" = "fail" ] && [ "$2" = "0" ]; then
1722         save="no"
1723     fi
1724
1725     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1726         dest="${ARTIFACT_DIRECTORY}/${testname:?}.journal"
1727     else
1728         dest="${TESTDIR:?}/system.journal"
1729     fi
1730
1731     for j in "${1:?}"/*; do
1732         if get_bool "$save"; then
1733             if [ "$SYSTEMD_JOURNAL_REMOTE" = "" ]; then
1734                 cp -a "$j" "$dest"
1735             else
1736                 "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $j"
1737             fi
1738         fi
1739
1740         if [ -n "${TEST_SHOW_JOURNAL}" ]; then
1741             echo "---- $j ----"
1742             "$JOURNALCTL" --no-pager -o short-monotonic --no-hostname --priority="${TEST_SHOW_JOURNAL}" -D "$j"
1743         fi
1744
1745         rm -r "$j"
1746     done
1747
1748     if ! get_bool "$save"; then
1749         return 0
1750     fi
1751
1752     if [ -n "${SUDO_USER}" ]; then
1753         setfacl -m "user:${SUDO_USER:?}:r-X" "$dest"*
1754     fi
1755
1756     # we want to print this sometime later, so save this in a variable
1757     JOURNAL_LIST="$(ls -l "$dest"*)"
1758 }
1759
1760 check_result_common() {
1761     local workspace="${1:?}"
1762     local ret
1763
1764     if [ -s "$workspace/failed" ]; then
1765         # Non-empty …/failed has highest priority
1766         cp -a "$workspace/failed" "${TESTDIR:?}/"
1767         if [ -n "${SUDO_USER}" ]; then
1768             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1769         fi
1770         ret=1
1771     elif get_bool "$TIMED_OUT"; then
1772         echo "(timeout)" >"${TESTDIR:?}/failed"
1773         ret=2
1774     elif [ -e "$workspace/testok" ]; then
1775         # …/testok always counts (but with lower priority than …/failed)
1776         ret=0
1777     elif [ -e "$workspace/skipped" ]; then
1778         # …/skipped always counts (a message is expected)
1779         echo "${TESTNAME:?} was skipped:"
1780         cat "$workspace/skipped"
1781         ret=0
1782     else
1783         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1784         ret=3
1785     fi
1786
1787     check_asan_reports "$workspace" || ret=4
1788
1789     check_coverage_reports "$workspace" || ret=5
1790
1791     save_journal "$workspace/var/log/journal" $ret
1792
1793     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1794         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1795     fi
1796
1797     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1798         echo -n "${TESTNAME:?}: "
1799         cat "$TESTDIR/failed"
1800     fi
1801     echo "${JOURNAL_LIST:-"No journals were saved"}"
1802
1803     return ${ret:?}
1804 }
1805
1806 check_result_nspawn() {
1807     local workspace="${1:?}"
1808     local ret=0
1809
1810     # Run a test-specific checks if defined by check_result_nspawn_hook()
1811     if declare -F check_result_nspawn_hook >/dev/null; then
1812         if ! check_result_nspawn_hook "${workspace}"; then
1813             derror "check_result_nspawn_hook() returned with EC > 0"
1814             ret=4
1815         fi
1816     fi
1817
1818     check_result_common "${workspace}" || ret=$?
1819
1820     _umount_dir "${initdir:?}"
1821
1822     return $ret
1823 }
1824
1825 # can be overridden in specific test
1826 check_result_qemu() {
1827     local ret=0
1828     mount_initdir
1829
1830     # Run a test-specific checks if defined by check_result_qemu_hook()
1831     if declare -F check_result_qemu_hook >/dev/null; then
1832         if ! check_result_qemu_hook "${initdir:?}"; then
1833             derror "check_result_qemu_hook() returned with EC > 0"
1834             ret=4
1835         fi
1836     fi
1837
1838     check_result_common "${initdir:?}" || ret=$?
1839
1840     _umount_dir "${initdir:?}"
1841
1842     return $ret
1843 }
1844
1845 check_result_nspawn_unittests() {
1846     local workspace="${1:?}"
1847     local ret=1
1848
1849     [[ -e "$workspace/testok" ]] && ret=0
1850
1851     if [[ -s "$workspace/failed" ]]; then
1852         ret=$((ret + 1))
1853         echo "=== Failed test log ==="
1854         cat "$workspace/failed"
1855     else
1856         if [[ -s "$workspace/skipped" ]]; then
1857             echo "=== Skipped test log =="
1858             cat "$workspace/skipped"
1859             # We might have only skipped tests - that should not fail the job
1860             ret=0
1861         fi
1862         if [[ -s "$workspace/testok" ]]; then
1863             echo "=== Passed tests ==="
1864             cat "$workspace/testok"
1865         fi
1866     fi
1867
1868     get_bool "${TIMED_OUT:=}" && ret=1
1869     check_coverage_reports "$workspace" || ret=5
1870
1871     save_journal "$workspace/var/log/journal" $ret
1872
1873     _umount_dir "${initdir:?}"
1874
1875     return $ret
1876 }
1877
1878 check_result_qemu_unittests() {
1879     local ret=1
1880
1881     mount_initdir
1882     [[ -e "${initdir:?}/testok" ]] && ret=0
1883
1884     if [[ -s "$initdir/failed" ]]; then
1885         ret=$((ret + 1))
1886         echo "=== Failed test log ==="
1887         cat "$initdir/failed"
1888     else
1889         if [[ -s "$initdir/skipped" ]]; then
1890             echo "=== Skipped test log =="
1891             cat "$initdir/skipped"
1892             # We might have only skipped tests - that should not fail the job
1893             ret=0
1894         fi
1895         if [[ -s "$initdir/testok" ]]; then
1896             echo "=== Passed tests ==="
1897             cat "$initdir/testok"
1898         fi
1899     fi
1900
1901     get_bool "${TIMED_OUT:=}" && ret=1
1902     check_coverage_reports "$initdir" || ret=5
1903
1904     save_journal "$initdir/var/log/journal" $ret
1905
1906     _umount_dir "$initdir"
1907
1908     return $ret
1909 }
1910
1911 create_rc_local() {
1912     dinfo "Create rc.local"
1913     mkdir -p "${initdir:?}/etc/rc.d"
1914     cat >"$initdir/etc/rc.d/rc.local" <<EOF
1915 #!/usr/bin/env bash
1916 exit 0
1917 EOF
1918     chmod 0755 "$initdir/etc/rc.d/rc.local"
1919 }
1920
1921 install_execs() {
1922     ddebug "Install executables from the service files"
1923
1924     local pkg_config_path="${BUILD_DIR:?}/src/core/"
1925     local systemunitdir userunitdir exe
1926     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
1927     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
1928     while read -r exe; do
1929         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
1930         # also, plymouth is pulled in by rescue.service, but even there the exit code
1931         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
1932         dinfo "Attempting to install $exe (based on unit file reference)"
1933         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
1934     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
1935 }
1936
1937 generate_module_dependencies() {
1938     dinfo "Generate modules dependencies"
1939     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
1940         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
1941             dfatal "\"depmod -a $KERNEL_VER\" failed."
1942             exit 1
1943     fi
1944 }
1945
1946 install_depmod_files() {
1947     dinfo "Install depmod files"
1948     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
1949     inst "/lib/modules/$KERNEL_VER/modules.builtin"
1950 }
1951
1952 install_plymouth() {
1953     dinfo "Install plymouth"
1954     # install plymouth, if found... else remove plymouth service files
1955     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
1956     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
1957     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
1958     #         image_install plymouth plymouthd
1959     # else
1960         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
1961     # fi
1962 }
1963
1964 install_haveged() {
1965     # If haveged is installed, it's probably included in initrd and needs to be
1966     # installed in the image too.
1967     if [ -x /usr/sbin/haveged ]; then
1968         dinfo "Install haveged files"
1969         inst /usr/sbin/haveged
1970         for u in /usr/lib/systemd/system/haveged*; do
1971             inst "$u"
1972         done
1973     fi
1974 }
1975
1976 install_ld_so_conf() {
1977     dinfo "Install /etc/ld.so.conf*"
1978     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
1979     ldconfig -r "$initdir"
1980 }
1981
1982 install_testuser() {
1983     dinfo "Set up a test user"
1984     # create unprivileged user for user manager tests
1985     mkdir -p "${initdir:?}/etc/sysusers.d"
1986     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
1987 u testuser    4711     "Test User" /home/testuser
1988 EOF
1989
1990     mkdir -p "$initdir/home/testuser"
1991     chmod 0700 "$initdir/home/testuser"
1992     chown 4711:4711 "$initdir/home/testuser"
1993 }
1994
1995 install_config_files() {
1996     dinfo "Install config files"
1997     inst /etc/sysconfig/init || :
1998     inst /etc/passwd
1999     inst /etc/shadow
2000     inst_any /etc/login.defs /usr/etc/login.defs
2001     inst /etc/group
2002     inst /etc/shells
2003     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
2004     inst /etc/pam.conf || :
2005     inst_any /etc/os-release /usr/lib/os-release
2006     inst /etc/localtime
2007     # we want an empty environment
2008     : >"${initdir:?}/etc/environment"
2009     : >"$initdir/etc/machine-id"
2010     : >"$initdir/etc/resolv.conf"
2011
2012     # set the hostname
2013     echo 'H' >"$initdir/etc/hostname"
2014
2015     # let's set up just one image with the traditional verbose output
2016     if [ "${IMAGE_NAME:?}" != "basic" ]; then
2017         mkdir -p "$initdir/etc/systemd/system.conf.d"
2018         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
2019     fi
2020 }
2021
2022 install_basic_tools() {
2023     dinfo "Install basic tools"
2024     image_install "${BASICTOOLS[@]}"
2025     image_install -o sushell
2026     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
2027     image_install -o ldconfig.real
2028 }
2029
2030 install_debug_tools() {
2031     dinfo "Install debug tools"
2032     image_install -o "${DEBUGTOOLS[@]}"
2033
2034     if get_bool "$INTERACTIVE_DEBUG"; then
2035         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
2036         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
2037         mkdir -p "$getty_override"
2038         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
2039         echo 'export TERM=linux' >>"$initdir/etc/profile"
2040
2041         if command -v resize >/dev/null; then
2042             image_install resize
2043             echo "resize" >>"$initdir/etc/profile"
2044         fi
2045
2046         # Sometimes we might end up with plymouthd still running (especially
2047         # with the initrd -> asan_wrapper -> systemd transition), which will eat
2048         # our inputs and make debugging via tty impossible. Let's fix this by
2049         # killing plymouthd explicitly for the interactive sessions.
2050         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
2051         #   - https://github.com/llvm/llvm-project/issues/49223
2052         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
2053         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
2054         cat >"$plymouth_unit" <<EOF
2055 [Unit]
2056 After=multi-user.target
2057
2058 [Service]
2059 ExecStart=sh -c 'killall --verbose plymouthd || :'
2060
2061 [Install]
2062 WantedBy=multi-user.target
2063 EOF
2064         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
2065     fi
2066 }
2067
2068 install_libnss() {
2069     dinfo "Install libnss"
2070     # install libnss_files for login
2071     local NSS_LIBS
2072     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
2073     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
2074         image_install "${NSS_LIBS[@]}"
2075     fi
2076 }
2077
2078 install_dbus() {
2079     dinfo "Install dbus"
2080     inst "${ROOTLIBDIR:?}/system/dbus.socket"
2081
2082     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2083     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
2084         inst "$ROOTLIBDIR/system/dbus-broker.service"
2085         inst_symlink /etc/systemd/system/dbus.service
2086         inst /usr/bin/dbus-broker
2087         inst /usr/bin/dbus-broker-launch
2088     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
2089         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2090         inst "$ROOTLIBDIR/system/dbus-daemon.service"
2091         # Alias symlink
2092         inst_symlink /etc/systemd/system/dbus.service
2093     else
2094         inst "$ROOTLIBDIR/system/dbus.service"
2095     fi
2096
2097     while read -r file; do
2098         inst "$file"
2099     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
2100
2101     # setup policy for Type=dbus test
2102     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
2103     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
2104 <?xml version="1.0"?>
2105 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2106         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2107 <busconfig>
2108     <policy user="root">
2109         <allow own="systemd.test.ExecStopPost"/>
2110     </policy>
2111 </busconfig>
2112 EOF
2113
2114     # If we run without KVM, bump the service start timeout
2115     if ! get_bool "$QEMU_KVM"; then
2116         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
2117 <?xml version="1.0"?>
2118 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2119         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2120 <busconfig>
2121     <limit name="service_start_timeout">120000</limit>
2122 </busconfig>
2123 EOF
2124         # Bump the client-side timeout in sd-bus as well
2125         mkdir -p "$initdir/etc/systemd/system.conf.d"
2126         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=120' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
2127     fi
2128 }
2129
2130 install_user_dbus() {
2131     dinfo "Install user dbus"
2132     local userunitdir
2133     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
2134         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
2135         userunitdir=/usr/lib/systemd/user
2136     fi
2137
2138     inst "$userunitdir/dbus.socket"
2139     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
2140
2141     # Append the After= dependency on dbus in case it isn't already set up
2142     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
2143     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
2144 [Unit]
2145 After=dbus.service
2146 EOF
2147
2148     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2149     if [ -f "$userunitdir/dbus-broker.service" ]; then
2150         inst "$userunitdir/dbus-broker.service"
2151         inst_symlink /etc/systemd/user/dbus.service
2152     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
2153         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2154         inst "$userunitdir/dbus-daemon.service"
2155         # Alias symlink
2156         inst_symlink /etc/systemd/user/dbus.service
2157     else
2158         inst "$userunitdir/dbus.service"
2159     fi
2160 }
2161
2162 install_pam() {
2163     dinfo "Install PAM"
2164     local paths=()
2165
2166     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
2167         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
2168     else
2169         paths+=(/lib*/security)
2170     fi
2171
2172     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
2173         [ -d "$d" ] && paths+=("$d")
2174     done
2175
2176     while read -r file; do
2177         inst "$file"
2178     done < <(find "${paths[@]}" -xtype f)
2179
2180     # pam_unix depends on unix_chkpwd.
2181     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2182     image_install -o unix_chkpwd
2183
2184     # set empty root password for easy debugging
2185     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2186
2187     # And make sure pam_unix will accept it by making sure that
2188     # the PAM module has the nullok option.
2189     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2190         [ -d "$initdir/$d" ] || continue
2191         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2192     done
2193 }
2194
2195 install_locales() {
2196     # install only C.UTF-8 and English locales
2197     dinfo "Install locales"
2198
2199     if command -v meson >/dev/null \
2200             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2201             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2202         # locale-gen support
2203         image_install -o locale-gen localedef
2204         inst /etc/locale.gen || :
2205         inst /usr/share/i18n/SUPPORTED || :
2206         inst_recursive /usr/share/i18n/charmaps
2207         inst_recursive /usr/share/i18n/locales
2208         inst_recursive /usr/share/locale/en*
2209         inst_recursive /usr/share/locale/de*
2210         image_install /usr/share/locale/locale.alias
2211         # locale-gen might either generate each locale separately or merge them
2212         # into a single archive
2213         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2214               image_install /usr/lib/locale/locale-archive); then
2215             dfatal "Failed to install required locales"
2216             exit 1
2217         fi
2218     else
2219         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2220     fi
2221 }
2222
2223 # shellcheck disable=SC2120
2224 install_keymaps() {
2225     local i p
2226     local -a prefix=(
2227         "/usr/lib"
2228         "/usr/share"
2229     )
2230
2231     dinfo "Install console keymaps"
2232
2233     if (( $# == 0 )); then
2234         for p in "${prefix[@]}"; do
2235             # The first three paths may be deprecated.
2236             # It seems now the last three paths are used by many distributions.
2237             for i in \
2238                 "$p"/kbd/keymaps/include/* \
2239                 "$p"/kbd/keymaps/i386/include/* \
2240                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2241                 "$p"/kbd/keymaps/legacy/include/* \
2242                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2243                 "$p"/kbd/keymaps/xkb/us*; do
2244                     [[ -f "$i" ]] || continue
2245                     inst "$i"
2246             done
2247         done
2248     else
2249         # When it takes any argument, then install more keymaps.
2250         for p in "${prefix[@]}"; do
2251             for i in \
2252                 "$p"/kbd/keymaps/include/* \
2253                 "$p"/kbd/keymaps/i386/*/* \
2254                 "$p"/kbd/keymaps/legacy/i386/*/* \
2255                 "$p"/kbd/keymaps/xkb/*; do
2256                     [[ -f "$i" ]] || continue
2257                     inst "$i"
2258             done
2259         done
2260     fi
2261 }
2262
2263 install_x11_keymaps() {
2264     dinfo "Install x11 keymaps"
2265
2266     if (( $# == 0 )); then
2267         # Install only keymap list.
2268         inst /usr/share/X11/xkb/rules/base.lst
2269     else
2270         # When it takes any argument, then install all keymaps.
2271         inst_recursive /usr/share/X11/xkb
2272     fi
2273 }
2274
2275 install_zoneinfo() {
2276     dinfo "Install time zones"
2277     inst_any /usr/share/zoneinfo/Asia/Seoul
2278     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2279     inst_any /usr/share/zoneinfo/Australia/Sydney
2280     inst_any /usr/share/zoneinfo/Europe/Berlin
2281     inst_any /usr/share/zoneinfo/Europe/Dublin
2282     inst_any /usr/share/zoneinfo/Europe/Kiev
2283     inst_any /usr/share/zoneinfo/Pacific/Auckland
2284     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2285     inst_any /usr/share/zoneinfo/CET
2286     inst_any /usr/share/zoneinfo/EET
2287     inst_any /usr/share/zoneinfo/UTC
2288 }
2289
2290 install_fonts() {
2291     dinfo "Install system fonts"
2292     for i in \
2293         /usr/{lib,share}/kbd/consolefonts/eurlatgr* \
2294         /usr/{lib,share}/kbd/consolefonts/latarcyrheb-sun16*; do
2295             [[ -f "$i" ]] || continue
2296             inst "$i"
2297     done
2298 }
2299
2300 install_terminfo() {
2301     dinfo "Install terminfo files"
2302     local terminfodir
2303     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2304         [ -f "${terminfodir}/l/linux" ] && break
2305     done
2306     image_install -o "${terminfodir}/l/linux"
2307 }
2308
2309 has_user_dbus_socket() {
2310     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2311         return 0
2312     else
2313         echo "Per-user instances are not supported. Skipping..."
2314         return 1
2315     fi
2316 }
2317
2318 setup_nspawn_root_hook() { :;}
2319
2320 setup_nspawn_root() {
2321     if [ -z "${initdir}" ]; then
2322         dfatal "\$initdir not defined"
2323         exit 1
2324     fi
2325
2326     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2327
2328     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2329         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2330         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2331     fi
2332
2333     setup_nspawn_root_hook
2334 }
2335
2336 setup_basic_dirs() {
2337     mkdir -p "${initdir:?}/run"
2338     mkdir -p "$initdir/etc/systemd/system"
2339     mkdir -p "$initdir/var/log/journal"
2340
2341
2342     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2343         if [ -L "/$d" ]; then
2344             inst_symlink "/$d"
2345         else
2346             inst_dir "/$d"
2347         fi
2348     done
2349
2350     ln -sfn /run "$initdir/var/run"
2351     ln -sfn /run/lock "$initdir/var/lock"
2352 }
2353
2354 mask_supporting_services() {
2355     # mask some services that we do not want to run in these tests
2356     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2357     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2358     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2359     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2360     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2361 }
2362
2363 inst_libs() {
2364     local bin="${1:?}"
2365     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2366     local file line
2367
2368     while read -r line; do
2369         [[ "$line" = 'not a dynamic executable' ]] && break
2370         # Ignore errors about our own stuff missing. This is most likely caused
2371         # by ldd attempting to use the unprefixed RPATH.
2372         [[ "$line" =~ (libsystemd|libudev).*\ not\ found ]] && continue
2373
2374         if [[ "$line" =~ not\ found ]]; then
2375             dfatal "Missing a shared library required by $bin."
2376             dfatal "Run \"ldd $bin\" to find out what it is."
2377             dfatal "$line"
2378             dfatal "Cannot create a test image."
2379             exit 1
2380         fi
2381
2382         if [[ "$line" =~ $so_regex ]]; then
2383             file="${BASH_REMATCH[1]}"
2384             [[ -e "${initdir:?}/$file" ]] && continue
2385             inst_library "$file"
2386         fi
2387     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2388 }
2389
2390 import_testdir() {
2391     # make sure we don't get a stale LOOPDEV value from old times
2392     local _LOOPDEV="${LOOPDEV:=}"
2393     # We don't want shellcheck to follow & check the $STATEFILE
2394     # shellcheck source=/dev/null
2395     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2396     LOOPDEV="$_LOOPDEV"
2397     if [[ ! -d "$TESTDIR" ]]; then
2398         if [[ -z "$TESTDIR" ]]; then
2399             TESTDIR="$(mktemp --tmpdir=$WORKDIR -d -t systemd-test.XXXXXX)"
2400         else
2401             mkdir -p "$TESTDIR"
2402         fi
2403
2404         cat >"$STATEFILE" <<EOF
2405 TESTDIR="$TESTDIR"
2406 EOF
2407         export TESTDIR
2408     fi
2409
2410     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2411     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2412 }
2413
2414 import_initdir() {
2415     initdir="${TESTDIR:?}/root"
2416     mkdir -p "$initdir"
2417     export initdir
2418 }
2419
2420 get_cgroup_hierarchy() {
2421     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2422         cgroup2fs)
2423             echo "unified"
2424             ;;
2425         tmpfs)
2426             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2427                 echo "hybrid"
2428             else
2429                 echo "legacy"
2430             fi
2431             ;;
2432         *)
2433             dfatal "Failed to determine host's cgroup hierarchy"
2434             exit 1
2435     esac
2436 }
2437
2438 ## @brief Converts numeric logging level to the first letter of level name.
2439 #
2440 # @param lvl Numeric logging level in range from 1 to 6.
2441 # @retval 1 if @a lvl is out of range.
2442 # @retval 0 if @a lvl is correct.
2443 # @result Echoes first letter of level name.
2444 _lvl2char() {
2445     case "$1" in
2446         1) echo F;;
2447         2) echo E;;
2448         3) echo W;;
2449         4) echo I;;
2450         5) echo D;;
2451         6) echo T;;
2452         *) return 1;;
2453     esac
2454 }
2455
2456 ## @brief Internal helper function for _do_dlog()
2457 #
2458 # @param lvl Numeric logging level.
2459 # @param msg Message.
2460 # @retval 0 It's always returned, even if logging failed.
2461 #
2462 # @note This function is not supposed to be called manually. Please use
2463 # dtrace(), ddebug(), or others instead which wrap this one.
2464 #
2465 # This function calls _do_dlog() either with parameter msg, or if
2466 # none is given, it will read standard input and will use every line as
2467 # a message.
2468 #
2469 # This enables:
2470 # dwarn "This is a warning"
2471 # echo "This is a warning" | dwarn
2472 LOG_LEVEL="${LOG_LEVEL:-4}"
2473
2474 dlog() {
2475     local lvl lvlc
2476
2477     [ -z "$LOG_LEVEL" ] && return 0
2478     lvl="${1:?}"; shift
2479     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2480     lvlc="$(_lvl2char "$lvl")" || return 0
2481
2482     if [ $# -ge 1 ]; then
2483         echo "$lvlc: $*"
2484     else
2485         while read -r line; do
2486             echo "$lvlc: " "$line"
2487         done
2488     fi
2489 }
2490
2491 ## @brief Logs message at TRACE level (6)
2492 #
2493 # @param msg Message.
2494 # @retval 0 It's always returned, even if logging failed.
2495 dtrace() {
2496     set +x
2497     dlog 6 "$@"
2498     if get_bool "${debug:=}"; then
2499         set -x
2500     fi
2501 }
2502
2503 ## @brief Logs message at DEBUG level (5)
2504 #
2505 # @param msg Message.
2506 # @retval 0 It's always returned, even if logging failed.
2507 ddebug() {
2508     dlog 5 "$@"
2509 }
2510
2511 ## @brief Logs message at INFO level (4)
2512 #
2513 # @param msg Message.
2514 # @retval 0 It's always returned, even if logging failed.
2515 dinfo() {
2516     set +x
2517     dlog 4 "$@"
2518     if get_bool "${debug:=}"; then
2519         set -x
2520     fi
2521 }
2522
2523 ## @brief Logs message at WARN level (3)
2524 #
2525 # @param msg Message.
2526 # @retval 0 It's always returned, even if logging failed.
2527 dwarn() {
2528     set +x
2529     dlog 3 "$@"
2530     if get_bool "${debug:=}"; then
2531         set -x
2532     fi
2533 }
2534
2535 ## @brief Logs message at ERROR level (2)
2536 #
2537 # @param msg Message.
2538 # @retval 0 It's always returned, even if logging failed.
2539 derror() {
2540     dlog 2 "$@"
2541 }
2542
2543 ## @brief Logs message at FATAL level (1)
2544 #
2545 # @param msg Message.
2546 # @retval 0 It's always returned, even if logging failed.
2547 dfatal() {
2548     set +x
2549     dlog 1 "$@"
2550     if get_bool "${debug:=}"; then
2551         set -x
2552     fi
2553 }
2554
2555
2556 # Generic substring function.  If $2 is in $1, return 0.
2557 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2558
2559 # normalize_path <path>
2560 # Prints the normalized path, where it removes any duplicated
2561 # and trailing slashes.
2562 # Example:
2563 # $ normalize_path ///test/test//
2564 # /test/test
2565 normalize_path() {
2566     shopt -q -s extglob
2567     set -- "${1//+(\/)//}"
2568     shopt -q -u extglob
2569     echo "${1%/}"
2570 }
2571
2572 # convert_abs_rel <from> <to>
2573 # Prints the relative path, when creating a symlink to <to> from <from>.
2574 # Example:
2575 # $ convert_abs_rel /usr/bin/test /bin/test-2
2576 # ../../bin/test-2
2577 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2578 convert_abs_rel() {
2579     local __current __absolute __abssize __cursize __newpath
2580     local -i __i __level
2581
2582     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2583
2584     # corner case #1 - self looping link
2585     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2586
2587     # corner case #2 - own dir link
2588     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2589
2590     IFS="/" read -ra __current <<< "$1"
2591     IFS="/" read -ra __absolute <<< "$2"
2592
2593     __abssize=${#__absolute[@]}
2594     __cursize=${#__current[@]}
2595
2596     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2597         (( __level++ ))
2598         if (( __level > __abssize || __level > __cursize ))
2599         then
2600             break
2601         fi
2602     done
2603
2604     for ((__i = __level; __i < __cursize-1; __i++)); do
2605         if ((__i > __level))
2606         then
2607             __newpath=$__newpath"/"
2608         fi
2609         __newpath=$__newpath".."
2610     done
2611
2612     for ((__i = __level; __i < __abssize; __i++)); do
2613         if [[ -n $__newpath ]]
2614         then
2615             __newpath=$__newpath"/"
2616         fi
2617         __newpath=$__newpath${__absolute[__i]}
2618     done
2619
2620     echo "$__newpath"
2621 }
2622
2623
2624 # Install a directory, keeping symlinks as on the original system.
2625 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2626 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2627 # and a symlink ${initdir}/lib -> lib64.
2628 inst_dir() {
2629     local dir="${1:?}"
2630     local part="${dir%/*}"
2631     local file
2632
2633     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2634
2635     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2636         dir="$part $dir"
2637         part="${part%/*}"
2638     done
2639
2640     # iterate over parent directories
2641     for file in $dir; do
2642         [[ -e "${initdir}/$file" ]] && continue
2643         if [[ -L $file ]]; then
2644             inst_symlink "$file"
2645         else
2646             # create directory
2647             mkdir -m 0755 "${initdir}/$file" || return 1
2648             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2649             chmod u+w "${initdir}/$file"
2650         fi
2651     done
2652 }
2653
2654 # $1 = file to copy to ramdisk
2655 # $2 (optional) Name for the file on the ramdisk
2656 # Location of the image dir is assumed to be $initdir
2657 # We never overwrite the target if it exists.
2658 inst_simple() {
2659     [[ -f "${1:?}" ]] || return 1
2660     strstr "$1" "/" || return 1
2661
2662     local src="$1"
2663     local target="${2:-$1}"
2664     if ! [[ -d ${initdir:?}/$target ]]; then
2665         [[ -e ${initdir}/$target ]] && return 0
2666         [[ -L ${initdir}/$target ]] && return 0
2667         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2668     fi
2669     # install checksum files also
2670     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2671         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2672     fi
2673     ddebug "Installing $src"
2674     cp --sparse=always --force --dereference --preserve=all "$src" "${initdir}/$target"
2675 }
2676
2677 # find symlinks linked to given library file
2678 # $1 = library file
2679 # Function searches for symlinks by stripping version numbers appended to
2680 # library filename, checks if it points to the same target and finally
2681 # prints the list of symlinks to stdout.
2682 #
2683 # Example:
2684 # rev_lib_symlinks libfoo.so.8.1
2685 # output: libfoo.so.8 libfoo.so
2686 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2687 rev_lib_symlinks() {
2688     local fn="${1:?}"
2689     local links=""
2690     local orig
2691     orig="$(readlink -f "$1")"
2692
2693     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2694
2695     until [[ "${fn##*.}" == so ]]; do
2696         fn="${fn%.*}"
2697         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2698     done
2699
2700     echo "${links}"
2701 }
2702
2703 # Same as above, but specialized to handle dynamic libraries.
2704 # It handles making symlinks according to how the original library
2705 # is referenced.
2706 inst_library() {
2707     local src="${1:?}"
2708     local dest="${2:-$1}"
2709     local reallib symlink
2710
2711     strstr "$1" "/" || return 1
2712     [[ -e ${initdir:?}/$dest ]] && return 0
2713     if [[ -L $src ]]; then
2714         # install checksum files also
2715         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2716             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2717         fi
2718         reallib="$(readlink -f "$src")"
2719         inst_simple "$reallib" "$reallib"
2720         inst_dir "${dest%/*}"
2721         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2722         ddebug "Creating symlink $reallib -> $dest"
2723         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2724     else
2725         inst_simple "$src" "$dest"
2726     fi
2727
2728     # Create additional symlinks.  See rev_symlinks description.
2729     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2730         if [[ ! -e "$initdir/$symlink" ]]; then
2731             ddebug "Creating extra symlink: $symlink"
2732             inst_symlink "$symlink"
2733         fi
2734     done
2735 }
2736
2737 # find a binary.  If we were not passed the full path directly,
2738 # search in the usual places to find the binary.
2739 find_binary() {
2740     local bin="${1:?}"
2741     if [[ -z ${bin##/*} ]]; then
2742         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2743             echo "$bin"
2744             return 0
2745         fi
2746     fi
2747
2748     type -P "$bin"
2749 }
2750
2751 # Same as above, but specialized to install binary executables.
2752 # Install binary executable, and all shared library dependencies, if any.
2753 inst_binary() {
2754     local bin="${1:?}"
2755     local path target
2756
2757     # In certain cases we might attempt to install a binary which is already
2758     # present in the test image, yet it's missing from the host system.
2759     # In such cases, let's check if the binary indeed exists in the image
2760     # before doing any other checks. If it does, immediately return with
2761     # success.
2762     if [[ $# -eq 1 ]]; then
2763         for path in "" bin sbin usr/bin usr/sbin; do
2764             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2765         done
2766     fi
2767
2768     bin="$(find_binary "$bin")" || return 1
2769     target="${2:-$bin}"
2770     [[ -e "${initdir:?}/$target" ]] && return 0
2771     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2772
2773     local file line
2774     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2775     # DSOs provided by systemd
2776     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2777     local wrap_binary=0
2778     local enable_lsan=0
2779     # I love bash!
2780     while read -r line; do
2781         [[ "$line" = 'not a dynamic executable' ]] && break
2782
2783         # Ignore errors about our own stuff missing. This is most likely caused
2784         # by ldd attempting to use the unprefixed RPATH.
2785         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2786
2787         # We're built with ASan and the target binary loads one of the systemd's
2788         # DSOs, so we need to tweak the environment before executing the binary
2789         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2790             wrap_binary=1
2791         fi
2792
2793         if [[ "$line" =~ $so_regex ]]; then
2794             file="${BASH_REMATCH[1]}"
2795             [[ -e "${initdir}/$file" ]] && continue
2796             inst_library "$file"
2797             continue
2798         fi
2799
2800         if [[ "$line" =~ not\ found ]]; then
2801             dfatal "Missing a shared library required by $bin."
2802             dfatal "Run \"ldd $bin\" to find out what it is."
2803             dfatal "$line"
2804             dfatal "Cannot create a test image."
2805             exit 1
2806         fi
2807     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2808
2809     # Same as above, but we need to wrap certain libraries unconditionally
2810     #
2811     # chown, getent, login, setfacl, su, useradd, userdel
2812     #    - dlopen() (not only) systemd's PAM modules
2813     # ls, mkfs.*, mksquashfs, mkswap, setpriv, stat
2814     #   - pull in nss_systemd with certain options (like ls -l) when
2815     #     nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2816     # delv, dig - pull in nss_resolve if `resolve` is in nsswitch.conf
2817     # tar - called by machinectl in TEST-25
2818     bin_rx='/(agetty|chown|curl|delv|dig|getfacl|getent|id|login|ls|mkfs\.[a-z0-9]+|mksquashfs|mkswap|setfacl|setpriv|stat|su|tar|useradd|userdel)$'
2819     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ $bin_rx ]]; then
2820         wrap_binary=1
2821         # Ugh, so we want to disable LSan in most cases for the wrapped binaries, since
2822         # we don't care about memory leaks in such binaries. However, in certain cases
2823         # the external binary is the only interface for the systemd code, like for
2824         # the systemd NSS modules, where we want to detect memory leaks. So let's
2825         # do another check to decide if we want to enable LSan for given binary.
2826         if [[ "$bin" =~ /getent$ ]]; then
2827             enable_lsan=1
2828         fi
2829     fi
2830
2831     # If the target binary is built with ASan support, we don't need to wrap
2832     # it, as it should handle everything by itself
2833     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2834         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2835         # Install the target binary with a ".orig" suffix
2836         inst_simple "$bin" "${target}.orig"
2837         # Create a simple shell wrapper in place of the target binary, which
2838         # sets necessary ASan-related env variables and then exec()s the
2839         # suffixed target binary
2840         cat >"$initdir/$target" <<EOF
2841 #!/bin/bash
2842 # Preload the ASan runtime DSO, otherwise ASAn will complain
2843 export LD_PRELOAD="$ASAN_RT_PATH"
2844 # Disable LSan to speed things up, since we don't care about leak reports
2845 # from 'external' binaries
2846 export ASAN_OPTIONS=detect_leaks=$enable_lsan
2847 # Set argv[0] to the original binary name without the ".orig" suffix
2848 exec -a "\$0" -- "${target}.orig" "\$@"
2849 EOF
2850         chmod +x "$initdir/$target"
2851     else
2852         inst_simple "$bin" "$target"
2853     fi
2854 }
2855
2856 # same as above, except for shell scripts.
2857 # If your shell script does not start with shebang, it is not a shell script.
2858 inst_script() {
2859     local bin line shebang_regex
2860     bin="$(find_binary "${1:?}")" || return 1
2861     shift
2862
2863     read -r -n 80 line <"$bin"
2864     # If debug is set, clean unprintable chars to prevent messing up the term
2865     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2866     shebang_regex='(#! *)(/[^ ]+).*'
2867     [[ "$line" =~ $shebang_regex ]] || return 1
2868     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2869 }
2870
2871 # same as above, but specialized for symlinks
2872 inst_symlink() {
2873     local src="${1:?}"
2874     local target="${2:-$src}"
2875     local realsrc
2876
2877     strstr "$src" "/" || return 1
2878     [[ -L "$src" ]] || return 1
2879     [[ -L "${initdir:?}/$target" ]] && return 0
2880     realsrc="$(readlink -f "$src")"
2881     if ! [[ -e "$initdir/$realsrc" ]]; then
2882         if [[ -d "$realsrc" ]]; then
2883             inst_dir "$realsrc"
2884         else
2885             inst "$realsrc"
2886         fi
2887     fi
2888     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
2889     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
2890     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
2891 }
2892
2893 # attempt to install any programs specified in a udev rule
2894 inst_rule_programs() {
2895     local rule="${1:?}"
2896     local prog bin
2897
2898     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
2899         if [ -x "/lib/udev/$prog" ]; then
2900             bin="/lib/udev/$prog"
2901         else
2902             if ! bin="$(find_binary "$prog")"; then
2903                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
2904                 continue
2905             fi
2906         fi
2907
2908         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
2909         image_install "$bin"
2910     done
2911 }
2912
2913 # udev rules always get installed in the same place, so
2914 # create a function to install them to make life simpler.
2915 inst_rules() {
2916     local target=/etc/udev/rules.d
2917     local found rule
2918
2919     inst_dir "/lib/udev/rules.d"
2920     inst_dir "$target"
2921     for rule in "$@"; do
2922         if [ "${rule#/}" = "$rule" ]; then
2923             for r in /lib/udev/rules.d /etc/udev/rules.d; do
2924                 if [[ -f "$r/$rule" ]]; then
2925                     found="$r/$rule"
2926                     inst_simple "$found"
2927                     inst_rule_programs "$found"
2928                 fi
2929             done
2930         fi
2931         for r in '' ./; do
2932             if [[ -f "${r}${rule}" ]]; then
2933                 found="${r}${rule}"
2934                 inst_simple "$found" "$target/${found##*/}"
2935                 inst_rule_programs "$found"
2936             fi
2937         done
2938         [[ $found ]] || dinfo "Skipping udev rule: $rule"
2939         found=
2940     done
2941 }
2942
2943 # general purpose installation function
2944 # Same args as above.
2945 inst() {
2946     case $# in
2947         1) ;;
2948         2)
2949             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
2950             [[ "$initdir" = "$2" ]] && set "$1"
2951             ;;
2952         3)
2953             [[ -z "$initdir" ]] && export initdir="$2"
2954             set "$1" "$3"
2955             ;;
2956         *)
2957             dfatal "inst only takes 1 or 2 or 3 arguments"
2958             exit 1
2959             ;;
2960     esac
2961
2962     local fun
2963     for fun in inst_symlink inst_script inst_binary inst_simple; do
2964         "$fun" "$@" && return 0
2965     done
2966
2967     dwarn "Failed to install '$1'"
2968     return 1
2969 }
2970
2971 # install any of listed files
2972 #
2973 # If first argument is '-d' and second some destination path, first accessible
2974 # source is installed into this path, otherwise it will installed in the same
2975 # path as source.  If none of listed files was installed, function return 1.
2976 # On first successful installation it returns with 0 status.
2977 #
2978 # Example:
2979 #
2980 # inst_any -d /bin/foo /bin/bar /bin/baz
2981 #
2982 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
2983 # initrd.
2984 inst_any() {
2985     local dest file
2986
2987     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
2988
2989     for file in "$@"; do
2990         if [[ -e "$file" ]]; then
2991             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
2992             inst "$file" && return 0
2993         fi
2994     done
2995
2996     return 1
2997 }
2998
2999 inst_recursive() {
3000     local p item
3001
3002     for p in "$@"; do
3003         # Make sure the source exists, as the process substitution below
3004         # suppresses errors
3005         stat "$p" >/dev/null || return 1
3006
3007         while read -r item; do
3008             if [[ -d "$item" ]]; then
3009                 inst_dir "$item"
3010             elif [[ -f "$item" ]]; then
3011                 inst_simple "$item"
3012             fi
3013         done < <(find "$p" 2>/dev/null)
3014     done
3015 }
3016
3017 # image_install [-o ] <file> [<file> ... ]
3018 # Install <file> to the test image
3019 # -o optionally install the <file> and don't fail, if it is not there
3020 image_install() {
3021     local optional=no
3022     local prog="${1:?}"
3023
3024     if [[ "$prog" = '-o' ]]; then
3025         optional=yes
3026         shift
3027     fi
3028
3029     for prog in "$@"; do
3030         if ! inst "$prog" ; then
3031             if get_bool "$optional"; then
3032                 dinfo "Skipping program $prog as it cannot be found and is" \
3033                     "flagged to be optional"
3034             else
3035                 dfatal "Failed to install $prog"
3036                 exit 1
3037             fi
3038         fi
3039     done
3040 }
3041
3042 # Install a single kernel module along with any firmware it may require.
3043 # $1 = full path to kernel module to install
3044 install_kmod_with_fw() {
3045     local module="${1:?}"
3046     # no need to go further if the module is already installed
3047     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
3048     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
3049
3050     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
3051
3052     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
3053
3054     local modname="${module##*/}"
3055     local fwdir found fw
3056     modname="${modname%.ko*}"
3057
3058     while read -r fw; do
3059         found=
3060         for fwdir in /lib/firmware/updates /lib/firmware; do
3061             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
3062                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
3063                 found=yes
3064             fi
3065         done
3066         if ! get_bool "$found"; then
3067             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
3068                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
3069                     "\"${modname}.ko\""
3070             else
3071                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
3072                     "\"${modname}.ko\""
3073             fi
3074         fi
3075     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
3076     return 0
3077 }
3078
3079 # Do something with all the dependencies of a kernel module.
3080 # Note that kernel modules depend on themselves using the technique we use
3081 # $1 = function to call for each dependency we find
3082 #      It will be passed the full path to the found kernel module
3083 # $2 = module to get dependencies for
3084 # rest of args = arguments to modprobe
3085 for_each_kmod_dep() {
3086     local func="${1:?}"
3087     local kmod="${2:?}"
3088     local found=0
3089     local cmd modpath
3090     shift 2
3091
3092     while read -r cmd modpath _; do
3093         [[ "$cmd" = insmod ]] || continue
3094         "$func" "$modpath" || return $?
3095         found=1
3096     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
3097
3098     ! get_bool "$found" && return 1
3099     return 0
3100 }
3101
3102 # instmods [-c] <kernel module> [<kernel module> ... ]
3103 # instmods [-c] <kernel subsystem>
3104 # install kernel modules along with all their dependencies.
3105 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
3106 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
3107 instmods() {
3108     local check=no
3109     if [[ $# -ge 0 && "$1" = '-c' ]]; then
3110         check=yes
3111         shift
3112     fi
3113
3114     inst1mod() {
3115         local mod="${1:?}"
3116         local ret=0
3117         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
3118
3119         case "$mod" in
3120             =*)
3121                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
3122                     (
3123                         [[ "$mpargs" ]] && echo "$mpargs"
3124                         cat "${mod_dir}/modules.${mod#=}"
3125                     ) | instmods
3126                 else
3127                     (
3128                         [[ "$mpargs" ]] && echo "$mpargs"
3129                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
3130                     ) | instmods
3131                 fi
3132                 ;;
3133             --*)
3134                 mpargs+=" $mod"
3135                 ;;
3136             i2o_scsi)
3137                 # Do not load this diagnostic-only module
3138                 return
3139                 ;;
3140             *)
3141                 mod=${mod##*/}
3142                 # if we are already installed, skip this module and go on
3143                 # to the next one.
3144                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
3145
3146                 # We use '-d' option in modprobe only if modules prefix path
3147                 # differs from default '/'.  This allows us to use Dracut with
3148                 # old version of modprobe which doesn't have '-d' option.
3149                 local mod_dirname=${mod_dir%%/lib/modules/*}
3150                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
3151
3152                 # ok, load the module, all its dependencies, and any firmware
3153                 # it may require
3154                 for_each_kmod_dep install_kmod_with_fw "$mod" \
3155                     --set-version "$KERNEL_VER" \
3156                     ${mod_dirname:+"$mod_dirname"} \
3157                     ${mpargs:+"$mpargs"}
3158                 ((ret+=$?))
3159                 ;;
3160         esac
3161         return "$ret"
3162     }
3163
3164     local mod mpargs
3165
3166     if [[ $# -eq 0 ]]; then  # filenames from stdin
3167         while read -r mod; do
3168             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3169                 dfatal "Failed to install $mod"
3170                 return 1
3171             fi
3172         done
3173     fi
3174
3175     for mod in "$@"; do # filenames as arguments
3176         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3177             dfatal "Failed to install $mod"
3178             return 1
3179         fi
3180     done
3181
3182     return 0
3183 }
3184
3185 _umount_dir() {
3186     local mountpoint="${1:?}"
3187     if mountpoint -q "$mountpoint"; then
3188         ddebug "umount $mountpoint"
3189         umount "$mountpoint"
3190     fi
3191 }
3192
3193 # can be overridden in specific test
3194 test_setup_cleanup() {
3195     cleanup_initdir
3196 }
3197
3198 _test_cleanup() {
3199     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3200     (
3201         set +e
3202         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3203         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3204         # If multiple setups/cleans are ran in parallel, this can cause a race
3205         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3206             rm -vf "${IMAGESTATEDIR}/default.img"
3207         fi
3208         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3209         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3210     ) || :
3211 }
3212
3213 # can be overridden in specific test
3214 test_cleanup() {
3215     _test_cleanup
3216 }
3217
3218 test_cleanup_again() {
3219     [ -n "$TESTDIR" ] || return
3220     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3221     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3222 }
3223
3224 test_create_image() {
3225     create_empty_image_rootdir
3226
3227     # Create what will eventually be our root filesystem onto an overlay
3228     (
3229         LOG_LEVEL=5
3230         setup_basic_environment
3231     )
3232 }
3233
3234 test_setup() {
3235     if get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3236             command -v meson >/dev/null && \
3237             [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3238         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3239         exit 1
3240     fi
3241
3242     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3243         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3244         mount_initdir
3245     else
3246         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3247             # default.img is the base that every test uses and optionally appends to
3248             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3249                 # Create the backing public image, but then completely unmount
3250                 # it and drop the loopback device responsible for it, since we're
3251                 # going to symlink/copy the image and mount it again from
3252                 # elsewhere.
3253                 local image_old="${IMAGE_PUBLIC}"
3254                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3255                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3256                 fi
3257                 test_create_image
3258                 test_setup_cleanup
3259                 umount_loopback
3260                 cleanup_loopdev
3261                 IMAGE_PUBLIC="${image_old}"
3262             fi
3263             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3264                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3265             fi
3266         fi
3267
3268         local hook_defined
3269         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3270
3271         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3272         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3273             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3274         else
3275             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3276         fi
3277
3278         mount_initdir
3279
3280         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3281             dinfo "Masking supporting services"
3282             mask_supporting_services
3283         fi
3284
3285         # Send stdout/stderr of testsuite-*.service units to both journal and
3286         # console to make debugging in CIs easier
3287         # Note: we can't use a dropin for `testsuite-.service`, since that also
3288         #       overrides 'sub-units' of some tests that already use a specific
3289         #       value for Standard(Output|Error)=
3290         #       (e.g. test/units/testsuite-66-deviceisolation.service)
3291         if ! get_bool "$INTERACTIVE_DEBUG"; then
3292             local dropin_dir="${initdir:?}/etc/systemd/system/testsuite-${TESTID:?}.service.d"
3293             mkdir -p "$dropin_dir"
3294             printf '[Service]\nStandardOutput=journal+console\nStandardError=journal+console' >"$dropin_dir/99-stdout.conf"
3295         fi
3296
3297         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
3298             # Do an initial coverage capture, to make sure the final report includes
3299             # files that the tests didn't touch at all
3300             lcov --initial --capture --directory "${initdir}/${BUILD_DIR:?}" --output-file "${TESTDIR:?}/coverage-base"
3301         fi
3302
3303         if get_bool "$hook_defined"; then
3304             test_append_files "${initdir:?}"
3305         fi
3306     fi
3307
3308     setup_nspawn_root
3309 }
3310
3311 test_run() {
3312     local test_id="${1:?}"
3313     mount_initdir
3314
3315     # Reset the boot counter, if present
3316     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
3317
3318     if ! get_bool "${TEST_NO_QEMU:=}"; then
3319         if run_qemu "$test_id"; then
3320             check_result_qemu || { echo "qemu test failed"; return 1; }
3321         else
3322             dwarn "can't run qemu, skipping"
3323         fi
3324     fi
3325     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3326         mount_initdir
3327         if run_nspawn "${initdir:?}" "$test_id"; then
3328             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3329         else
3330             dwarn "can't run systemd-nspawn, skipping"
3331         fi
3332
3333         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3334             dir="$TESTDIR/unprivileged-nspawn-root"
3335             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3336                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3337             else
3338                 dwarn "can't run systemd-nspawn, skipping"
3339             fi
3340         fi
3341     fi
3342     return 0
3343 }
3344
3345 do_test() {
3346     if [[ $UID != "0" ]]; then
3347         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3348         exit 0
3349     fi
3350
3351     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3352         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3353         exit 0
3354     fi
3355
3356     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3357         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3358         exit 0
3359     fi
3360
3361     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3362         TEST_NO_QEMU=1
3363     fi
3364
3365     # Detect lib paths
3366     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3367         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3368     done
3369
3370     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3371         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3372     done
3373
3374     mkdir -p "$WORKDIR"
3375     mkdir -p "$STATEDIR"
3376
3377     import_testdir
3378     import_initdir
3379
3380     if [ -n "${SUDO_USER}" ]; then
3381         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3382         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3383     fi
3384
3385     testname="$(basename "$PWD")"
3386
3387     while (($# > 0)); do
3388         case $1 in
3389             --run)
3390                 echo "${testname} RUN: $TEST_DESCRIPTION"
3391                 test_run "$TESTID"
3392                 ret=$?
3393                 if [ $ret -eq 0 ]; then
3394                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3395                 else
3396                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3397                 fi
3398                 exit $ret
3399                 ;;
3400             --setup)
3401                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3402                 test_setup
3403                 test_setup_cleanup
3404                 ;;
3405             --clean)
3406                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3407                 test_cleanup
3408                 ;;
3409             --clean-again)
3410                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3411                 test_cleanup_again
3412                 ;;
3413             --all)
3414                 ret=0
3415                 echo -n "${testname}: $TEST_DESCRIPTION "
3416                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3417                 # and loop devices will leak
3418                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3419                 if [ $ret -eq 0 ]; then
3420                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3421                 fi
3422                 if [ $ret -eq 0 ]; then
3423                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3424                 fi
3425                 test_cleanup
3426                 if [ $ret -eq 0 ]; then
3427                     # $TESTLOG is in $STATEDIR, so clean it up only on success
3428                     [[ -n "$STATEDIR" ]] && rm -vfr "$STATEDIR"
3429                     echo "[OK]"
3430                 else
3431                     echo "[FAILED]"
3432                     echo "see $TESTLOG"
3433                 fi
3434                 exit $ret
3435                 ;;
3436             *)
3437                 break
3438                 ;;
3439         esac
3440         shift
3441     done
3442 }