test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 # Simple wrapper to unify boolean checks.
  18 # Note: this function needs to stay near the top of the file, so we can use it
  19 #       in code in the outermost scope.
  20 get_bool() {
  21     # Make the value lowercase to make the regex matching simpler
  22     local _bool="${1,,}"
  23
  24     # Consider empty value as "false"
  25     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  26         return 1
  27     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  28         return 0
  29     else
  30         echo >&2 "Value '$_bool' is not a valid boolean value"
  31         exit 1
  32     fi
  33 }
  34
  35 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  36 export PATH
  37
  38 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  39 # shellcheck source=/dev/null
  40 source "$os_release"
  41 [[ "$ID" == "debian" || " $ID_LIKE " == *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=no
  42 [[ "$ID" == "arch" || " $ID_LIKE " == *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=no
  43 [[ "$ID" == "fedora" ]] && LOOKS_LIKE_FEDORA=yes || LOOKS_LIKE_FEDORA=no
  44 [[ " $ID_LIKE " == *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=no
  45
  46 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  47 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  48 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  49 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  50 get_bool "$LOOKS_LIKE_SUSE" && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  51 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  52 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  53 # Note that defining a different IMAGE_NAME in a test setup script will only result
  54 # in default.img being copied and renamed. It can then be extended by defining
  55 # a test_append_files() function. The $1 parameter will be the root directory.
  56 # To force creating a new image from scratch (eg: to encrypt it), also define
  57 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  58 IMAGE_NAME=${IMAGE_NAME:-default}
  59 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  60 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  61 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  62 LOOPDEV=
  63
  64 # Since in Bash we can have only one handler per signal, let's overcome this
  65 # limitation by having one global handler for the EXIT signal which executes
  66 # all registered handlers
  67 _AT_EXIT_HANDLERS=()
  68 _at_exit() {
  69     set +e
  70
  71     # Run the EXIT handlers in reverse order
  72     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  73         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  74         eval "${_AT_EXIT_HANDLERS[$i]}"
  75     done
  76 }
  77
  78 trap _at_exit EXIT
  79
  80 add_at_exit_handler() {
  81     _AT_EXIT_HANDLERS+=("${1:?}")
  82 }
  83
  84 # Decide if we can (and want to) run qemu with KVM acceleration.
  85 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  86 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  87 # running under KVM. If these conditions are met, enable KVM (and possibly
  88 # nested KVM), otherwise disable it.
  89 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  90     QEMU_KVM=yes
  91 else
  92     QEMU_KVM=no
  93 fi
  94
  95 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  96     echo "WARNING! Cannot determine libdir from pkg-config, assuming /usr/lib/systemd" >&2
  97     ROOTLIBDIR=/usr/lib/systemd
  98 fi
  99
 100 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
 101 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 102 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 103 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 104 # These variables are used by test scripts
 105 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR
 106
 107 TOOLS_DIR="$SOURCE_DIR/tools"
 108 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 109 if get_bool "${NO_BUILD:=}"; then
 110     BUILD_DIR="$SOURCE_DIR"
 111 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 112     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 113     exit 1
 114 fi
 115
 116 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 117 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 118 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 119 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 120 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 121 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 122 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 123 SYSTEMD_ID128="${SYSTEMD_ID128:-$(command -v "$BUILD_DIR/systemd-id128" || command -v systemd-id128)}"
 124
 125 TESTFILE="${BASH_SOURCE[1]}"
 126 if [ -z "$TESTFILE" ]; then
 127     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 128     exit 1
 129 fi
 130 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 131
 132 WORKDIR="/var/tmp/systemd-tests"
 133 if get_bool "${NO_BUILD:=}"; then
 134     STATEDIR="$WORKDIR/$TESTNAME"
 135 else
 136     STATEDIR="$BUILD_DIR/test/$TESTNAME"
 137 fi
 138
 139 STATEFILE="$STATEDIR/.testdir"
 140 IMAGESTATEDIR="$STATEDIR/.."
 141 TESTLOG="$STATEDIR/test.log"
 142
 143 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 144     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 145     exit 1
 146 fi
 147 TESTID="${BASH_REMATCH[1]:?}"
 148
 149 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 150     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 151     exit 1
 152 fi
 153
 154 BASICTOOLS=(
 155     awk
 156     base64
 157     basename
 158     bash
 159     capsh
 160     cat
 161     chgrp
 162     chmod
 163     chown
 164     chroot
 165     cmp
 166     cp
 167     cryptsetup
 168     cut
 169     date
 170     dd
 171     dhclient
 172     diff
 173     dirname
 174     dmsetup
 175     echo
 176     env
 177     false
 178     find
 179     findmnt
 180     flock
 181     getconf
 182     getent
 183     getfacl
 184     getfattr
 185     grep
 186     grep
 187     gunzip
 188     gzip
 189     head
 190     hostname
 191     id
 192     ionice
 193     ip
 194     jq
 195     killall
 196     ldd
 197     ln
 198     ln
 199     loadkeys
 200     login
 201     losetup
 202     ls
 203     lsattr
 204     lsblk
 205     lz4cat
 206     mkdir
 207     mkfifo
 208     mknod
 209     mktemp
 210     modprobe
 211     mount
 212     mountpoint
 213     mv
 214     nc
 215     nproc
 216     ping
 217     pkill
 218     ps
 219     readlink
 220     realpath
 221     rev
 222     rm
 223     rmdir
 224     rmmod
 225     script
 226     sed
 227     seq
 228     setfacl
 229     setfattr
 230     setfont
 231     setpriv
 232     setsid
 233     sfdisk
 234     sh
 235     sleep
 236     sort
 237     stat
 238     stty
 239     su
 240     sulogin
 241     sysctl
 242     tail
 243     tar
 244     tee
 245     test
 246     timeout
 247     touch
 248     tr
 249     true
 250     truncate
 251     tty
 252     umount
 253     uname
 254     unshare
 255     useradd
 256     userdel
 257     wc
 258     whoami
 259     xargs
 260     xzcat
 261 )
 262
 263 DEBUGTOOLS=(
 264     df
 265     dmesg
 266     du
 267     free
 268     less
 269     strace
 270     vi
 271     /usr/libexec/vi
 272 )
 273
 274 is_built_with_asan() {
 275     local _bin="${1:?}"
 276
 277     if ! type -P objdump >/dev/null; then
 278         echo "Failed to find objdump, assuming systemd hasn't been built with ASAN."
 279         return 1
 280     fi
 281
 282     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 283     local _asan_calls
 284     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 285     if ((_asan_calls < 1000)); then
 286         return 1
 287     else
 288         return 0
 289     fi
 290 }
 291
 292 is_built_with_coverage() {
 293     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 294         return 1
 295     fi
 296
 297     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 298 }
 299
 300 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 301 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 302
 303 if get_bool "$IS_BUILT_WITH_ASAN"; then
 304     PATH_TO_INIT="$ROOTLIBDIR/systemd-under-asan"
 305     QEMU_MEM="${QEMU_MEM:-2G}"
 306     QEMU_SMP="${QEMU_SMP:-4}"
 307
 308     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 309     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 310         ASAN_COMPILER=gcc
 311         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 312     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 313         ASAN_COMPILER=clang
 314         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 315
 316         # As clang's ASan DSO is usually in a non-standard path, let's check if
 317         # the environment is set accordingly. If not, warn the user and exit.
 318         # We're not setting the LD_LIBRARY_PATH automagically here, because
 319         # user should encounter (and fix) the same issue when running the unit
 320         # tests (meson test)
 321         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 322             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 323             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 324             exit 1
 325         fi
 326     else
 327         echo >&2 "systemd is not linked against the ASan DSO"
 328         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 329         exit 1
 330     fi
 331
 332     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 333 fi
 334
 335 test_require_bin() {
 336     local bin
 337
 338     for bin in "$@"; do
 339         if ! command -v "$bin" >/dev/null; then
 340             echo "Required binary $bin not available, skipping the test"
 341             exit 0
 342         fi
 343     done
 344 }
 345
 346 find_qemu_bin() {
 347     QEMU_BIN="${QEMU_BIN:-""}"
 348     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 349     if get_bool "$QEMU_KVM"; then
 350         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 351     fi
 352
 353     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 354     case $ARCH in
 355     x86_64)
 356         # QEMU's own build system calls it qemu-system-x86_64
 357         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 358         ;;
 359     i*86)
 360         # new i386 version of QEMU
 361         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 362
 363         # i386 version of QEMU
 364         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 365         ;;
 366     ppc64*)
 367         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 368         ;;
 369     esac
 370
 371     if [[ ! -e "$QEMU_BIN" ]]; then
 372         echo "Could not find a suitable qemu binary" >&2
 373         return 1
 374     fi
 375 }
 376
 377 qemu_setup_swtpm_socket() {
 378     local pid state_dir tpm_device
 379
 380     if ! tpm_device="$(qemu_get_tpm_device)"; then
 381         dinfo "Found QEMU version is too old for TPM2 on ppc64le"
 382         exit 0
 383     fi
 384
 385     state_dir="$(mktemp -d)"
 386     swtpm socket --tpm2 --tpmstate dir="$state_dir" --ctrl type=unixio,path="$state_dir/sock" &
 387     pid=$!
 388     if ! kill -0 "$pid"; then
 389         derror "Failed to start swtpm"
 390         return 1
 391     fi
 392
 393     if ! timeout 5 bash -c "until [[ -S $state_dir/sock ]]; do sleep .5; done"; then
 394         derror "Failed to setup swtpm socket"
 395         return 1
 396     fi
 397
 398     dinfo "Started swtpm as PID $pid with state dir $state_dir"
 399
 400     add_at_exit_handler "kill -TERM $pid 2>/dev/null; rm -rf '$state_dir'"
 401
 402     QEMU_OPTIONS+=" -chardev socket,id=chrtpm,path=$state_dir/sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device $tpm_device,tpmdev=tpm0"
 403     dinfo "Configured emulated TPM2 device $tpm_device"
 404
 405     return 0
 406 }
 407
 408 qemu_get_tpm_device() {
 409     local tpm_device="tpm-tis"
 410
 411     if [[ "$(uname -m)" == "ppc64le" ]]; then
 412         # tpm-spapr support was introduced in qemu 5.0.0
 413         if ! qemu_min_version "5.0.0"; then
 414             return 1
 415         fi
 416
 417         tpm_device="tpm-spapr"
 418     fi
 419
 420     echo "$tpm_device"
 421     return 0
 422 }
 423
 424 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 425 # returns 0 if newer or equal
 426 # returns 1 if older
 427 # returns 2 if failing
 428 qemu_min_version() {
 429     find_qemu_bin || return 2
 430
 431     # get version from binary
 432     local qemu_ver
 433     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 434
 435     # Check version string format
 436     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 437     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 438
 439     # compare as last command to return that value
 440     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 441 }
 442
 443 # Pads a file to multiple of 4 bytes
 444 pad4_file() {
 445     local size
 446     size=$(stat -c "%s" "$1")
 447     local padded
 448     padded=$((((size + 3) / 4) * 4))
 449     truncate -s "$padded" "$1"
 450 }
 451
 452 # Return 0 if qemu did run (then you must check the result state/logs for actual
 453 # success), or 1 if qemu is not available.
 454 run_qemu() {
 455     if declare -F run_qemu_hook >/dev/null; then
 456         if ! run_qemu_hook "${workspace}"; then
 457             derror "check_qemu_hook() returned with EC > 0"
 458             ret=4
 459         fi
 460     fi
 461
 462     # If the test provided its own initrd, use it (e.g. TEST-24)
 463     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 464         INITRD="$TESTDIR/initrd.img"
 465     fi
 466
 467     if [ -f /etc/machine-id ]; then
 468         read -r MACHINE_ID </etc/machine-id
 469         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 470             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 471         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 472             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 473     fi
 474
 475     local CONSOLE=ttyS0
 476
 477     # Reset the boot counter, if present
 478     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 479     rm -f "$initdir"/{testok,failed,skipped}
 480     # make sure the initdir is not mounted to avoid concurrent access
 481     cleanup_initdir
 482     umount_loopback
 483
 484     if [[ ! "$KERNEL_BIN" ]]; then
 485         if get_bool "$LOOKS_LIKE_ARCH"; then
 486             KERNEL_BIN=/boot/vmlinuz-linux
 487         else
 488             [ "$ARCH" ] || ARCH=$(uname -m)
 489             case $ARCH in
 490                 ppc64*)
 491                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 492                     # uses the "standard" vmlinuz- prefix
 493                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 494                     CONSOLE=hvc0
 495                     ;;
 496                 *)
 497                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 498                     ;;
 499             esac
 500         fi
 501     fi
 502
 503     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 504     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 505     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 506     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 507     if [[ ! "$INITRD" ]]; then
 508         if [[ -e "$default_fedora_initrd" ]]; then
 509             INITRD="$default_fedora_initrd"
 510         elif get_bool "$LOOKS_LIKE_DEBIAN" && [[ -e "$default_debian_initrd" ]]; then
 511             INITRD="$default_debian_initrd"
 512         elif get_bool "$LOOKS_LIKE_ARCH" && [[ -e "$default_arch_initrd" ]]; then
 513             INITRD="$default_arch_initrd"
 514         elif get_bool "$LOOKS_LIKE_SUSE" && [[ -e "$default_suse_initrd" ]]; then
 515             INITRD="$default_suse_initrd"
 516         fi
 517     fi
 518
 519     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 520     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 521     # is not installed or there's some other error when calling it, fall back
 522     # to the original value (QEMU_SMP=1).
 523     if [[ -z "${QEMU_SMP:=}" ]]; then
 524         if ! QEMU_SMP=$(nproc); then
 525             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 526             QEMU_SMP=1
 527         fi
 528     fi
 529
 530     find_qemu_bin || return 1
 531
 532     if get_bool "${TEST_SETUP_SWTPM:-}"; then
 533         qemu_setup_swtpm_socket || return 1
 534     fi
 535
 536     # Umount initdir to avoid concurrent access to the filesystem
 537     _umount_dir "$initdir"
 538
 539     local kernel_params=()
 540     local qemu_options=()
 541     local qemu_cmd=("$QEMU_BIN")
 542
 543     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 544         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 545     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 546         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 547     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 548         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 549     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 550         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 551         exit 1
 552     fi
 553
 554     if get_bool "$LOOKS_LIKE_SUSE"; then
 555         kernel_params+=("rd.hostonly=0")
 556     fi
 557
 558     # Debian/Ubuntu's initramfs tries to check if it can resume from hibernation
 559     # and wastes a minute or so probing disks, skip that as it's not useful here
 560     kernel_params+=(
 561         "root=LABEL=systemd_boot"
 562         "rw"
 563         "raid=noautodetect"
 564         "rd.luks=0"
 565         "loglevel=2"
 566         "init=$PATH_TO_INIT"
 567         "console=$CONSOLE"
 568         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 569         "systemd.unit=testsuite.target"
 570         "systemd.wants=testsuite-$1.service"
 571         "noresume"
 572         "oops=panic"
 573         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 574         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 575     )
 576
 577     if ! get_bool "$INTERACTIVE_DEBUG" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 578         kernel_params+=(
 579             "panic=1"
 580             "softlockup_panic=1"
 581             "systemd.wants=end.service"
 582         )
 583     fi
 584
 585     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 586     qemu_options+=(
 587         -smp "$QEMU_SMP"
 588         -net none
 589         -m "${QEMU_MEM:-768M}"
 590         -nographic
 591         -kernel "$KERNEL_BIN"
 592         -drive "format=raw,cache=unsafe,file=$image"
 593         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 594     )
 595
 596     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 597         local user_qemu_options
 598         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 599         qemu_options+=("${user_qemu_options[@]}")
 600     fi
 601     qemu_options+=(${QEMU_OPTIONS_ARRAY:+"${QEMU_OPTIONS_ARRAY[@]}"})
 602
 603     if [[ -n "$INITRD" ]]; then
 604         if [[ -n "$INITRD_EXTRA" ]]; then
 605             # An addition initrd has been specified, let's combine it with the main one.
 606             local t="$WORKDIR"/initrd.combined."$RANDOM"
 607
 608             # First, show contents of additional initrd
 609             echo "Additional initrd contents:"
 610             cpio -tv < "$INITRD_EXTRA"
 611
 612             # Copy the main initrd
 613             zstd -d -c -f "$INITRD" > "$t"
 614             add_at_exit_handler "rm $t"
 615             # Kernel requires this to be padded to multiple of 4 bytes with zeroes
 616             pad4_file "$t"
 617
 618             # Copy the additional initrd
 619             cat "$INITRD_EXTRA" >> "$t"
 620             pad4_file "$t"
 621
 622             qemu_options+=(-initrd "$t")
 623         else
 624             qemu_options+=(-initrd "$INITRD")
 625         fi
 626     fi
 627
 628     # Let's use KVM if possible
 629     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 630         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 631     fi
 632
 633     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 634         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 635     fi
 636
 637     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]} ${KERNEL_APPEND:-}" |& tee "${TESTDIR:?}/console.log")
 638     rc=$?
 639     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 640         derror "Test timed out after ${QEMU_TIMEOUT}s"
 641         TIMED_OUT=1
 642     else
 643         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 644     fi
 645     return 0
 646 }
 647
 648 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 649 # success), or 1 if nspawn is not available.
 650 run_nspawn() {
 651     [[ -d /run/systemd/system ]] || return 1
 652     # Reset the boot counter, if present
 653     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 654     rm -f "${initdir:?}"/{testok,failed,skipped}
 655
 656     local nspawn_cmd=()
 657     local nspawn_options=(
 658         "--register=no"
 659         "--kill-signal=SIGKILL"
 660         "--directory=${1:?}"
 661         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 662         "--machine=TEST-$TESTID"
 663     )
 664     local kernel_params=(
 665         "$PATH_TO_INIT"
 666         "systemd.unit=testsuite.target"
 667         "systemd.wants=testsuite-$2.service"
 668         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 669         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 670     )
 671
 672     if get_bool "$INTERACTIVE_DEBUG"; then
 673         nspawn_options+=("--console=interactive")
 674     elif ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 675         kernel_params+=("systemd.wants=end.service")
 676     fi
 677
 678     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 679         local user_nspawn_arguments
 680         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 681         nspawn_options+=("${user_nspawn_arguments[@]}")
 682     fi
 683
 684     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 685         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 686         exit
 687     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 688         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 689     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 690         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 691     else
 692         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 693         exit 1
 694     fi
 695
 696     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 697         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 698     else
 699         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 700     fi
 701
 702     # Word splitting here is intentional
 703     # shellcheck disable=SC2086
 704     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}" ${KERNEL_APPEND:-} |& tee "${TESTDIR:?}/console.log")
 705     rc=$?
 706     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 707         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 708         TIMED_OUT=1
 709     else
 710         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 711     fi
 712     return 0
 713 }
 714
 715 # Build two very minimal root images, with two units, one is the same and one is different across them
 716 install_verity_minimal() {
 717     dinfo "Set up a set of minimal images for verity verification"
 718     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 719         return
 720     fi
 721     if ! command -v mksquashfs >/dev/null 2>&1; then
 722         dfatal "mksquashfs not found"
 723         exit 1
 724     fi
 725     if ! command -v veritysetup >/dev/null 2>&1; then
 726         dfatal "veritysetup not found"
 727         exit 1
 728     fi
 729     # Local modifications of some global variables is intentional in this
 730     # subshell (SC2030)
 731     # shellcheck disable=SC2030
 732     (
 733         BASICTOOLS=(
 734             bash
 735             cat
 736             grep
 737             mount
 738             sleep
 739             touch
 740         )
 741         oldinitdir="$initdir"
 742         rm -rfv "$TESTDIR/minimal"
 743         export initdir="$TESTDIR/minimal"
 744         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 745         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 746         setup_basic_dirs
 747         install_basic_tools
 748         install_ld_so_conf
 749         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 750         # issue, thus falsely triggering SC2030 in this case
 751         # See: koalaman/shellcheck#1409
 752         if [[ -v ASAN_RT_PATH ]]; then
 753             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 754             # into the verity images to get rid of the annoying errors about
 755             # missing $LD_PRELOAD libraries.
 756             inst_libs "$ASAN_RT_PATH"
 757             inst_library "$ASAN_RT_PATH"
 758         fi
 759         cp "$os_release" "$initdir/usr/lib/os-release"
 760         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 761         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 762         touch "$initdir/opt/some_file"
 763         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 764         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 765         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 766 [Service]
 767 ExecStartPre=cat /usr/lib/os-release
 768 ExecStart=sleep 120
 769 EOF
 770         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 771
 772         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 773         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 774             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 775
 776         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 777         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 778         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 779
 780         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 781         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 782             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 783
 784         # Rolling distros like Arch do not set VERSION_ID
 785         local version_id=""
 786         if grep -q "^VERSION_ID=" "$os_release"; then
 787             version_id="$(grep "^VERSION_ID=" "$os_release")"
 788         fi
 789
 790         export initdir="$TESTDIR/app0"
 791         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 792         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 793         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 794         ( echo "${version_id}"
 795           echo "SYSEXT_IMAGE_ID=app" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 796         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 797 [Service]
 798 Type=oneshot
 799 RemainAfterExit=yes
 800 ExecStart=/opt/script0.sh
 801 TemporaryFileSystem=/var/lib
 802 StateDirectory=app0
 803 RuntimeDirectory=app0
 804 EOF
 805         cat >"$initdir/opt/script0.sh" <<EOF
 806 #!/bin/bash
 807 set -e
 808 test -e /usr/lib/os-release
 809 echo bar >\${STATE_DIRECTORY}/foo
 810 cat /usr/lib/extension-release.d/extension-release.app0
 811 EOF
 812         chmod +x "$initdir/opt/script0.sh"
 813         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 814         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 815
 816         export initdir="$TESTDIR/conf0"
 817         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system" "$initdir/opt"
 818         grep "^ID=" "$os_release" >"$initdir/etc/extension-release.d/extension-release.conf0"
 819         echo "${version_id}" >>"$initdir/etc/extension-release.d/extension-release.conf0"
 820         ( echo "${version_id}"
 821           echo "CONFEXT_IMAGE_ID=app" ) >>"$initdir/etc/extension-release.d/extension-release.conf0"
 822         echo MARKER_1 >"$initdir/etc/systemd/system/some_file"
 823         mksquashfs "$initdir" "$oldinitdir/usr/share/conf0.raw" -noappend
 824
 825         export initdir="$TESTDIR/app1"
 826         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 827         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 828         ( echo "${version_id}"
 829           echo "SYSEXT_SCOPE=portable"
 830           echo "SYSEXT_IMAGE_ID=app"
 831           echo "SYSEXT_IMAGE_VERSION=1"
 832           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 833         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 834         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 835 [Service]
 836 Type=oneshot
 837 RemainAfterExit=yes
 838 ExecStart=/opt/script1.sh
 839 StateDirectory=app1
 840 RuntimeDirectory=app1
 841 EOF
 842         cat >"$initdir/opt/script1.sh" <<EOF
 843 #!/bin/bash
 844 set -e
 845 test -e /usr/lib/os-release
 846 echo baz >\${STATE_DIRECTORY}/foo
 847 cat /usr/lib/extension-release.d/extension-release.app2
 848 EOF
 849         chmod +x "$initdir/opt/script1.sh"
 850         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 851         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 852
 853         export initdir="$TESTDIR/app-nodistro"
 854         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 855         ( echo "ID=_any"
 856           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 857         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 858         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 859
 860         export initdir="$TESTDIR/service-scoped-test"
 861         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system"
 862         ( echo "ID=_any"
 863           echo "ARCHITECTURE=_any" ) >"$initdir/etc/extension-release.d/extension-release.service-scoped-test"
 864         echo MARKER_CONFEXT_123 >"$initdir/etc/systemd/system/some_file"
 865         mksquashfs "$initdir" "$oldinitdir/etc/service-scoped-test.raw" -noappend
 866
 867         # We need to create a dedicated sysext image to test the reload mechanism. If we share an image to install the
 868         # 'foo.service' it will be loaded from another test run, which will impact the targeted test.
 869         export initdir="$TESTDIR/app-reload"
 870         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 871         ( echo "ID=_any"
 872           echo "ARCHITECTURE=_any"
 873           echo "EXTENSION_RELOAD_MANAGER=1" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-reload"
 874         mkdir -p "$initdir/usr/lib/systemd/system/multi-user.target.d"
 875         cat >"${initdir}/usr/lib/systemd/system/foo.service" <<EOF
 876 [Service]
 877 Type=oneshot
 878 RemainAfterExit=yes
 879 ExecStart=echo foo
 880
 881 [Install]
 882 WantedBy=multi-user.target
 883 EOF
 884         { echo "[Unit]"; echo "Upholds=foo.service"; } > "$initdir/usr/lib/systemd/system/multi-user.target.d/10-foo-service.conf"
 885         mksquashfs "$initdir" "$oldinitdir/usr/share/app-reload.raw" -noappend
 886     )
 887 }
 888
 889 setup_basic_environment() {
 890     # create the basic filesystem layout
 891     setup_basic_dirs
 892
 893     install_systemd
 894     install_missing_libraries
 895     install_config_files
 896     install_zoneinfo
 897     create_rc_local
 898     install_basic_tools
 899     install_libnss
 900     install_pam
 901     install_dbus
 902     install_fonts
 903     install_locales
 904     install_keymaps
 905     install_x11_keymaps
 906     install_terminfo
 907     install_execs
 908     install_fs_tools
 909     install_modules
 910     install_plymouth
 911     install_haveged
 912     install_debug_tools
 913     install_ld_so_conf
 914     install_testuser
 915     has_user_dbus_socket && install_user_dbus
 916     setup_selinux
 917     install_depmod_files
 918     generate_module_dependencies
 919     if get_bool "$IS_BUILT_WITH_ASAN"; then
 920         create_asan_wrapper
 921     fi
 922     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 923         install_verity_minimal
 924     fi
 925 }
 926
 927 setup_selinux() {
 928     dinfo "Setup SELinux"
 929     # don't forget KERNEL_APPEND='... selinux=1 ...'
 930     if ! get_bool "$SETUP_SELINUX"; then
 931         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 932         return 0
 933     fi
 934
 935     for dir in /etc/selinux /usr/share/selinux; do
 936         rm -rf "${initdir:?}/$dir"
 937         if ! cp -ar "$dir" "$initdir/$dir"; then
 938             dfatal "Failed to copy $dir"
 939             exit 1
 940         fi
 941     done
 942
 943     # We use a custom autorelabel service instead of the SELinux provided set
 944     # of units & a generator, since the generator overrides the default target
 945     # to the SELinux one when it detects /.autorelabel. However, we use
 946     # systemd.unit= on the kernel command cmdline which always takes precedence,
 947     # rendering all SELinux efforts useless. Also, pulling in selinux-autorelabel.service
 948     # explicitly doesn't work either, as it doesn't check for the presence of /.autorelabel
 949     # and does the relabeling unconditionally which always ends with a reboot, so
 950     # we end up in a reboot loop (and it also spews quite a lot of errors as it
 951     # wants /etc/fstab and dracut-initramfs-restore).
 952     touch "$initdir/.autorelabel"
 953     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 954     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 955
 956     # Tools requires by fixfiles
 957     image_install awk bash cat chcon expr egrep find grep head secon setfiles rm sort uname uniq
 958     image_install fixfiles getenforce load_policy selinuxenabled sestatus
 959 }
 960
 961 install_valgrind() {
 962     if ! type -p valgrind; then
 963         dfatal "Failed to install valgrind"
 964         exit 1
 965     fi
 966
 967     local valgrind_bins valgrind_libs valgrind_supp
 968
 969     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null |
 970                                        sed -r -n 's/execve\("([^"]*)".*/\1/p')
 971     image_install "${valgrind_bins[@]}"
 972
 973     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null |
 974                                        sed -r -n 's|.*calling init: (/.*vgpreload_.*)|\1|p')
 975     image_install "${valgrind_libs[@]}"
 976
 977     readarray -t valgrind_supp < <(strace -e open valgrind /bin/true 2>&1 >/dev/null |
 978                                        sed -r -n 's,open\("([^"]*(/debug[^"]*|\.supp))".*= [0-9].*,\1,p')
 979     image_install "${valgrind_supp[@]}"
 980 }
 981
 982 create_valgrind_wrapper() {
 983     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 984     ddebug "Create $valgrind_wrapper"
 985     cat >"$valgrind_wrapper" <<EOF
 986 #!/usr/bin/env bash
 987
 988 mount -t proc proc /proc
 989 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 990 EOF
 991     chmod 0755 "$valgrind_wrapper"
 992 }
 993
 994 create_asan_wrapper() {
 995     local asan_wrapper default_asan_options default_ubsan_options default_environment manager_environment
 996
 997     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 998
 999     default_asan_options="${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1}"
1000     default_ubsan_options="${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}"
1001
1002     if [[ "$ASAN_COMPILER" == "clang" ]]; then
1003         # clang: install llvm-symbolizer to generate useful reports
1004         # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
1005         image_install "llvm-symbolizer"
1006
1007         # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
1008         # unnecessary for gcc & libasan, however, for clang this is crucial, as its
1009         # runtime ASan DSO is in a non-standard (library) path.
1010         mkdir -p "${initdir:?}/etc/ld.so.conf.d/"
1011         echo "${ASAN_RT_PATH%/*}" >"${initdir:?}/etc/ld.so.conf.d/asan-path-override.conf"
1012         ldconfig -r "$initdir"
1013     fi
1014
1015     # Create a simple environment file which can be included by systemd services
1016     # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
1017     cat >"${initdir:?}/usr/lib/systemd/systemd-asan-env" <<EOF
1018 LD_PRELOAD=$ASAN_RT_PATH
1019 ASAN_OPTIONS=$default_asan_options
1020 LSAN_OPTIONS=detect_leaks=0
1021 UBSAN_OPTIONS=$default_ubsan_options
1022 EOF
1023
1024     default_environment=(
1025         "ASAN_OPTIONS='$default_asan_options'"
1026         "UBSAN_OPTIONS='$default_ubsan_options'"
1027         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1028     )
1029     manager_environment=(
1030         "ASAN_OPTIONS='$default_asan_options:log_path=/systemd-pid1.asan.log:log_to_syslog=1'"
1031         "UBSAN_OPTIONS='$default_ubsan_options:log_path=/systemd-pid1.ubsan.log:log_to_syslog=1'"
1032         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1033     )
1034
1035     mkdir -p "${initdir:?}/etc/systemd/system.conf.d/"
1036     cat >"${initdir:?}/etc/systemd/system.conf.d/asan.conf" <<EOF
1037 [Manager]
1038 DefaultEnvironment=${default_environment[*]}
1039 ManagerEnvironment=${manager_environment[*]}
1040 DefaultTimeoutStartSec=180s
1041 EOF
1042
1043     # ASAN and syscall filters aren't compatible with each other.
1044     find "${initdir:?}" -name '*.service' -type f -print0 | xargs -0 sed -i 's/^\(MemoryDeny\|SystemCall\)/#\1/'
1045
1046     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journald.service.d/"
1047     cat >"${initdir:?}/etc/systemd/system/systemd-journald.service.d/asan-env.conf" <<EOF
1048 [Service]
1049 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
1050 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
1051 Environment=ASAN_OPTIONS=$default_asan_options:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=$default_ubsan_options:log_path=/systemd-journald.ubsan.log
1052
1053 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
1054 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
1055 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
1056 StandardOutput=file:/systemd-journald.out
1057 EOF
1058
1059     # 90s isn't enough for some services to finish when literally everything is run
1060     # under ASan+UBSan in containers, which, in turn, are run in VMs.
1061     # Let's limit which environments such services should be executed in.
1062     mkdir -p "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/"
1063     cat >"${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/asan.conf" <<EOF
1064 [Unit]
1065 ConditionVirtualization=container
1066
1067 [Service]
1068 TimeoutSec=240s
1069 EOF
1070
1071     # Let's override another hard-coded timeout that kicks in too early
1072     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/"
1073     cat >"${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/asan.conf" <<EOF
1074 [Service]
1075 TimeoutSec=180s
1076 EOF
1077
1078     asan_wrapper="${initdir:?}/${PATH_TO_INIT:?}"
1079     # Sanity check to make sure we don't overwrite something we shouldn't.
1080     [[ "$asan_wrapper" =~ systemd-under-asan$ ]]
1081
1082     cat >"$asan_wrapper" <<EOF
1083 #!/usr/bin/env bash
1084 set -eux
1085
1086 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
1087 export ${manager_environment[@]}
1088 [[ -n "\$ASAN_OPTIONS" && -n "\$UBSAN_OPTIONS" ]]
1089
1090 exec "$ROOTLIBDIR/systemd" "\$@"
1091 EOF
1092     chmod 0755 "$asan_wrapper"
1093 }
1094
1095 create_strace_wrapper() {
1096     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
1097     ddebug "Create $strace_wrapper"
1098     cat >"$strace_wrapper" <<EOF
1099 #!/usr/bin/env bash
1100
1101 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
1102 EOF
1103     chmod 0755 "$strace_wrapper"
1104 }
1105
1106 install_fs_tools() {
1107     dinfo "Install fsck"
1108     image_install /sbin/fsck*
1109     image_install -o /bin/fsck*
1110
1111     # fskc.reiserfs calls reiserfsck. so, install it
1112     image_install -o reiserfsck
1113
1114     # we use mkfs in system-repart tests
1115     image_install /sbin/mkfs.ext4
1116     image_install /sbin/mkfs.vfat
1117 }
1118
1119 install_modules() {
1120     dinfo "Install modules"
1121
1122     instmods bridge dummy ext4 ipvlan macvlan vfat veth
1123     instmods loop =block
1124     instmods nls_ascii =nls
1125     instmods overlay =overlayfs
1126     instmods scsi_debug
1127
1128     if get_bool "$LOOKS_LIKE_SUSE"; then
1129         instmods af_packet
1130     fi
1131 }
1132
1133 install_dmevent() {
1134     instmods dm_crypt =crypto
1135     inst_binary dmeventd
1136     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
1137     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1138         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
1139         # and since buster/bionic 95-dm-notify.rules
1140         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
1141         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
1142     else
1143         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
1144     fi
1145     if get_bool "$LOOKS_LIKE_SUSE"; then
1146         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
1147     fi
1148 }
1149
1150 install_multipath() {
1151     instmods "=md" multipath
1152     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
1153     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
1154     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1155         # Note: try both 60-kpartx.rules (as seen on Debian Sid with 0.9.4-7) and 95-kpartx.rules (as seen on
1156         # Ubuntu Jammy with 0.8.8-1ubuntu1.22.04.4)
1157         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-kpartx.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
1158     else
1159         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
1160     fi
1161     mkdir -p "${initdir:?}/etc/multipath"
1162
1163     local file
1164     while read -r file; do
1165         # Install libraries required by the given library
1166         inst_libs "$file"
1167         # Install the library itself and create necessary symlinks
1168         inst_library "$file"
1169     done < <(find /lib*/multipath -type f)
1170 }
1171
1172 install_lvm() {
1173     local lvm_rules rule_prefix
1174
1175     image_install lvm
1176     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1177     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1178     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1179
1180     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1181         lvm_rules="56-lvm.rules"
1182         rule_prefix=""
1183     else
1184         lvm_rules="11-dm-lvm.rules"
1185         rule_prefix="dm-"
1186     fi
1187
1188     # Support the new udev autoactivation introduced in lvm 2.03.14
1189     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1190     # Static autoactivation (via lvm2-activation-generator) was dropped
1191     # in lvm 2.03.15
1192     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1193     if [[ -f "/lib/udev/rules.d/69-${rule_prefix}lvm.rules" ]]; then
1194         inst_rules "$lvm_rules" "69-${rule_prefix}lvm.rules"
1195     else
1196         image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1197         image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1198         inst_rules "$lvm_rules" "69-${rule_prefix}lvm-metad.rules"
1199     fi
1200
1201     mkdir -p "${initdir:?}/etc/lvm"
1202 }
1203
1204 host_has_btrfs() (
1205     set -e
1206     modprobe -nv btrfs && command -v mkfs.btrfs && command -v btrfs || return $?
1207 )
1208
1209 install_btrfs() {
1210     instmods btrfs
1211     # Not all utilities provided by btrfs-progs are listed here; extend the list
1212     # if necessary
1213     image_install btrfs btrfstune mkfs.btrfs
1214     inst_rules 64-btrfs-dm.rules
1215 }
1216
1217 install_iscsi() {
1218     # Install both client and server side stuff by default
1219     local inst="${1:-}"
1220     local file
1221
1222     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1223     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1224     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1225         image_install iscsi-iname iscsiadm iscsid iscsistart
1226         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1227         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1228         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1229         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1230         mkdir -p "${initdir:?}/etc/iscsi"
1231         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1232         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1233         # a one-time service instead of distro package's post-install scripts.
1234         # However, some distros still use this approach even after this patch,
1235         # so prefer the already existing initiatorname.iscsi file if it exists.
1236         #
1237         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1238         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1239             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1240             if get_bool "$IS_BUILT_WITH_ASAN"; then
1241                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1242                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1243                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1244                 # (among others) pre-loads ASan's DSO.
1245                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1246                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1247             fi
1248         else
1249             inst_simple "/etc/iscsi/initiatorname.iscsi"
1250         fi
1251     fi
1252
1253     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1254     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1255     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1256         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1257         image_install -o /etc/sysconfig/tgtd
1258         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1259         mkdir -p "${initdir:?}/etc/tgt"
1260         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1261         # Install perl modules required by tgt-admin
1262         #
1263         # Forgive me father for I have sinned. The monstrosity below appends
1264         # a perl snippet to the `tgt-admin` perl script on the fly, which
1265         # dumps a list of files (perl modules) required by `tgt-admin` at
1266         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1267         # passed to `inst_simple` which installs the necessary files into the image
1268         #
1269         # shellcheck disable=SC2016
1270         while read -r file; do
1271             inst_simple "$file"
1272         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1273     fi
1274 }
1275
1276 host_has_mdadm() (
1277     set -e
1278     command -v mdadm || return $?
1279 )
1280
1281 install_mdadm() {
1282     local unit
1283     local mdadm_units=(
1284         system/mdadm-grow-continue@.service
1285         system/mdadm-last-resort@.service
1286         system/mdadm-last-resort@.timer
1287         system/mdmon@.service
1288         system/mdmonitor-oneshot.service
1289         system/mdmonitor-oneshot.timer
1290         system/mdmonitor.service
1291         system-shutdown/mdadm.shutdown
1292     )
1293
1294     instmods "=md"
1295     image_install mdadm mdmon
1296     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1297     # Fedora/CentOS/RHEL ships this rule file
1298     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1299
1300     for unit in "${mdadm_units[@]}"; do
1301         image_install "${ROOTLIBDIR:?}/$unit"
1302     done
1303
1304     # Disable the mdmonitor service, since it fails if there's no valid email address
1305     # configured in /etc/mdadm.conf, which just unnecessarily pollutes the logs
1306     "${SYSTEMCTL:?}" mask --root "${initdir:?}" mdmonitor.service || :
1307 }
1308
1309 install_compiled_systemd() {
1310     dinfo "Install compiled systemd"
1311
1312     local ninja_bin
1313     ninja_bin="$(type -P ninja || type -P ninja-build)"
1314     if [[ -z "$ninja_bin" ]]; then
1315         dfatal "ninja was not found"
1316         exit 1
1317     fi
1318     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1319
1320     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1321     # find them in the same directory as the runtime data counts
1322     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1323         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1324         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1325         # Set effective & default ACLs for the build dir so unprivileged
1326         # processes can write gcda files with coverage stats
1327         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1328     fi
1329 }
1330
1331 install_package_file() {
1332     local file="${1:?}"
1333
1334     # Skip missing files (like /etc/machine-info)
1335     [[ ! -e "$file" ]] && return 0
1336     # Skip python unit tests, since the image_install machinery will try to pull
1337     # in the whole python stack in a very questionable state, making the tests fail.
1338     # And given we're trying to transition to mkosi-based images anyway I'm not even
1339     # going to bother
1340     [[ "$file" =~ /tests/unit-tests/.*.py$ ]] && return 0
1341     # If the current file is a directory, create it with the original
1342     # mode; if it's a symlink to a directory, copy it as-is
1343     if [[ -d "$file" ]]; then
1344         inst_dir "$file"
1345     else
1346         inst "$file"
1347     fi
1348 }
1349
1350 install_debian_systemd() {
1351     dinfo "Install debian systemd"
1352
1353     local deb file
1354
1355     while read -r deb; do
1356         ddebug "Install debian files from package $deb"
1357         while read -r file; do
1358             install_package_file "$file"
1359         done < <(dpkg-query -L "$deb" 2>/dev/null)
1360     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1361 }
1362
1363 install_rpm() {
1364     local rpm="${1:?}"
1365     local file
1366
1367     if ! rpm -q "$rpm" >/dev/null; then
1368         derror "RPM $rpm is not installed"
1369         return 1
1370     fi
1371
1372     dinfo "Installing contents of RPM $rpm"
1373     while read -r file; do
1374         install_package_file "$file"
1375     done < <(rpm -ql "$rpm")
1376 }
1377
1378 install_suse_systemd() {
1379     local pkgs
1380
1381     dinfo "Install basic filesystem structure"
1382     install_rpm filesystem
1383
1384     dinfo "Install SUSE systemd"
1385
1386     pkgs=(
1387         systemd
1388         systemd-boot
1389         systemd-container
1390         systemd-coredump
1391         systemd-experimental
1392         systemd-homed
1393         systemd-journal-remote
1394         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1395         # systemd-network sub-package on openSUSE, has its own testsuite.
1396         systemd-network
1397         systemd-portable
1398         udev
1399     )
1400
1401     for p in "${pkgs[@]}"; do
1402         rpm -q "$p" &>/dev/null || continue
1403
1404         install_rpm "$p"
1405     done
1406
1407     dinfo "Install the data needed by the tests at runtime"
1408     inst_recursive "${SOURCE_DIR}/testdata"
1409     inst_recursive "${SOURCE_DIR}/unit-tests/manual"
1410
1411     # On openSUSE, this directory is not created at package install, at least
1412     # for now.
1413     mkdir -p "$initdir/var/log/journal/remote"
1414 }
1415
1416 install_fedora_systemd() {
1417     local required_packages=(
1418         systemd
1419         systemd-container
1420         systemd-libs
1421         systemd-pam
1422         systemd-tests
1423         systemd-udev
1424     )
1425     local optional_packages=(
1426         systemd-boot-unsigned
1427         systemd-bootchart
1428         systemd-journal-remote
1429         systemd-networkd
1430         systemd-oomd-defaults
1431         systemd-resolved
1432     )
1433     local package
1434
1435     for package in "${required_packages[@]}"; do
1436         install_rpm "$package"
1437     done
1438
1439     for package in "${optional_packages[@]}"; do
1440         rpm -q "$package" >/dev/null || continue
1441         install_rpm "$package"
1442     done
1443 }
1444
1445 install_distro_systemd() {
1446     dinfo "Install distro systemd"
1447
1448     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1449         install_debian_systemd
1450     elif get_bool "$LOOKS_LIKE_SUSE"; then
1451         install_suse_systemd
1452     elif get_bool "$LOOKS_LIKE_FEDORA"; then
1453         install_fedora_systemd
1454     else
1455         dfatal "NO_BUILD not supported for this distro"
1456         exit 1
1457     fi
1458 }
1459
1460 install_systemd() {
1461     dinfo "Install systemd"
1462     if get_bool "$NO_BUILD"; then
1463         install_distro_systemd
1464     else
1465         install_compiled_systemd
1466     fi
1467
1468     # Remove unneeded documentation
1469     rm -fr "${initdir:?}"/usr/share/{man,doc}
1470
1471     # Enable debug logging in PID1
1472     mkdir -p "$initdir/etc/systemd/system.conf.d/"
1473     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/system.conf.d/10-log-level.conf"
1474     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1475         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf.d/99-log-level.conf"
1476     fi
1477     # Enable debug logging for user instances as well
1478     mkdir -p "$initdir/etc/systemd/user.conf.d/"
1479     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/user.conf.d/10-log-level.conf"
1480     # Store coredumps in journal
1481     mkdir -p "$initdir/etc/systemd/coredump.conf.d/"
1482     echo -ne "[Coredump]\nStorage=journal\n" >"$initdir/etc/systemd/coredump.conf.d/10-storage-journal.conf"
1483     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1484     mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
1485     echo -ne "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/99-SYSTEMD_UNIT_PATH.conf"
1486
1487     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1488     # images, since it prevents gcov to write the coverage reports (*.gcda
1489     # files)
1490     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1491         mkdir -p "$initdir/etc/systemd/system/service.d/"
1492         echo -ne "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1493         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image to make the coverage work with
1494         # units using DynamicUser=yes. Do this only for services with test- prefix and a couple of
1495         # known-to-use DynamicUser=yes services, as setting this system-wide has many undesirable
1496         # side-effects, as it creates its own namespace.
1497         for service in capsule@ test- systemd-journal-{gatewayd,upload}; do
1498             mkdir -p "$initdir/etc/systemd/system/$service.service.d/"
1499             echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/$service.service.d/99-gcov-rwpaths-override.conf"
1500         done
1501         # Ditto, but for the user daemon
1502         mkdir -p "$initdir/etc/systemd/user/test-.service.d/"
1503         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf"
1504         # Bind the $BUILD_DIR into nspawn containers that are executed using
1505         # machinectl. Unfortunately, the .nspawn files don't support drop-ins
1506         # so we have to inject the bind mount directly into
1507         # the systemd-nspawn@.service unit.
1508         cp "$initdir/usr/lib/systemd/system/systemd-nspawn@.service" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1509         sed -ri "s/^ExecStart=.+$/& --bind=${BUILD_DIR//\//\\\/}/" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1510         # Pass the $BUILD_DIR as $COVERAGE_BUILD_DIR env variable to the system
1511         # manager, similarly to what we do with $ASAN_RT_PATH during sanitized
1512         # builds
1513         mkdir -p "$initdir/etc/systemd/system.conf.d/"
1514         echo -ne "[Manager]\nDefaultEnvironment=COVERAGE_BUILD_DIR=$BUILD_DIR\n" >"$initdir/etc/systemd/system.conf.d/99-COVERAGE_BUILD_DIR.conf"
1515     fi
1516
1517     # If we're built with -Dportabled=false, tests with systemd-analyze
1518     # --profile will fail. Since we need just the profile (text) files, let's
1519     # copy them into the image if they don't exist there.
1520     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1521     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1522         dinfo "Couldn't find portable profiles in the test image"
1523         dinfo "Copying them directly from the source tree"
1524         mkdir -p "$portable_dir"
1525         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1526     fi
1527 }
1528
1529 get_ldpath() {
1530     local rpath
1531     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1532
1533     if [ -z "$rpath" ] ; then
1534         echo "$BUILD_DIR"
1535     else
1536         echo "$rpath"
1537     fi
1538 }
1539
1540 install_missing_libraries() {
1541     dinfo "Install missing libraries"
1542     # install possible missing libraries
1543     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/unit-tests/{,manual/,unsafe/}}*; do
1544         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1545     done
1546
1547     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1548     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1549     # if not present
1550     # See: https://github.com/systemd/systemd/pull/23858
1551     while read -r libgcc_s; do
1552         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1553     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1554
1555     local lib path
1556     # A number of dependencies is now optional via dlopen, so the install
1557     # script will not pick them up, since it looks at linkage.
1558     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1 libarchive libgcrypt libkmod; do
1559         ddebug "Searching for $lib via pkg-config"
1560         if pkg-config --exists "$lib"; then
1561                 path="$(pkg-config --variable=libdir "$lib")"
1562                 if [ -z "${path}" ]; then
1563                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1564                     continue
1565                 fi
1566
1567                 if ! [[ ${lib} =~ ^lib ]]; then
1568                         lib="lib${lib}"
1569                 fi
1570                 # p11-kit-1's .so doesn't have the API level in the name
1571                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1572                         lib="libp11-kit"
1573                 fi
1574                 # Some pkg-config files are broken and give out the wrong paths
1575                 # (eg: libcryptsetup), so just ignore them
1576                 inst_libs "${path}/${lib}.so" || true
1577                 inst_library "${path}/${lib}.so" || true
1578
1579                 if [[ "$lib" == "libxkbcommon" ]]; then
1580                     install_x11_keymaps full
1581                 fi
1582         else
1583             ddebug "$lib.pc not found, skipping"
1584             continue
1585         fi
1586     done
1587
1588     # Install extra openssl 3 stuff
1589     path="$(pkg-config --variable=libdir libcrypto)"
1590     inst_simple "${path}/ossl-modules/legacy.so" || true
1591     inst_simple "${path}/ossl-modules/fips.so" || true
1592     inst_simple "${path}/engines-3/afalg.so" || true
1593     inst_simple "${path}/engines-3/capi.so" || true
1594     inst_simple "${path}/engines-3/loader_attic.so" || true
1595     inst_simple "${path}/engines-3/padlock.so" || true
1596
1597     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1598     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1599     # shellcheck disable=SC2046
1600     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1601 }
1602
1603 cleanup_loopdev() {
1604     if [ -n "${LOOPDEV:=}" ]; then
1605         ddebug "losetup -d $LOOPDEV"
1606         losetup -d "${LOOPDEV}"
1607         unset LOOPDEV
1608     fi
1609 }
1610
1611 add_at_exit_handler cleanup_loopdev
1612
1613 create_empty_image() {
1614     if [[ -z "${IMAGE_NAME:=}" ]]; then
1615         echo "create_empty_image: \$IMAGE_NAME not set"
1616         exit 1
1617     fi
1618
1619     # Partition sizes are in MiBs
1620     local root_size=768
1621     local data_size=100
1622     local esp_size=128
1623     local boot_size=128
1624     local total=
1625     if ! get_bool "$NO_BUILD"; then
1626         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1627             root_size=$((root_size + 200))
1628         fi
1629         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1630             root_size=$((root_size + 200))
1631         fi
1632         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1633             root_size=$((root_size + 250))
1634         fi
1635         if get_bool "$IS_BUILT_WITH_ASAN"; then
1636             root_size=$((root_size * 2))
1637         fi
1638     fi
1639
1640     if [[ "${IMAGE_ADDITIONAL_ROOT_SIZE:-0}" -gt 0 ]]; then
1641         root_size=$((root_size + IMAGE_ADDITIONAL_ROOT_SIZE))
1642     fi
1643     if [[ "${IMAGE_ADDITIONAL_DATA_SIZE:-0}" -gt 0 ]]; then
1644         data_size=$((data_size + IMAGE_ADDITIONAL_DATA_SIZE))
1645     fi
1646
1647     total=$((root_size + data_size + esp_size + boot_size))
1648
1649     echo "Setting up ${IMAGE_PUBLIC:?} (${total} MB)"
1650     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1651
1652     # Create the blank file to use as a root filesystem
1653     truncate -s "${total}M" "$IMAGE_PUBLIC"
1654
1655     LOOPDEV="$(losetup --show -P -f "$IMAGE_PUBLIC")"
1656     [[ -b "$LOOPDEV" ]] || return 1
1657     # Create two partitions - a root one and a data one (utilized by some tests)
1658     sfdisk "$LOOPDEV" <<EOF
1659 label: gpt
1660 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B name=esp  size=${esp_size}M
1661 type=$("${SYSTEMD_ID128:?}" show root -Pu) name=root size=${root_size}M bootable
1662 type=BC13C2FF-59E6-4262-A352-B275FD6F7172 name=boot size=${boot_size}M
1663 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1664 EOF
1665
1666     udevadm settle
1667
1668     if ! mkfs -t vfat "${LOOPDEV}p1"; then
1669         dfatal "Failed to mkfs -t vfat ${LOOPDEV}p1"
1670         exit 1
1671     fi
1672
1673     local label=(-L systemd_boot)
1674     # mkfs.reiserfs doesn't know -L. so, use --label instead
1675     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1676     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p2" -q; then
1677         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p2 -q"
1678         exit 1
1679     fi
1680
1681     local label=(-L xbootldr)
1682     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label xbootldr)
1683     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p3" -q; then
1684         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p3 -q"
1685         exit 1
1686     fi
1687 }
1688
1689 mount_initdir() {
1690     if [ -z "${LOOPDEV:=}" ]; then
1691         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1692         LOOPDEV="$(losetup --show -P -f "$image")"
1693         [ -b "$LOOPDEV" ] || return 1
1694
1695         udevadm settle
1696     fi
1697
1698     if ! mountpoint -q "${initdir:?}"; then
1699         mkdir -p "$initdir"
1700         mount "${LOOPDEV}p2" "$initdir"
1701         TEST_SETUP_CLEANUP_ROOTDIR=1
1702     fi
1703 }
1704
1705 cleanup_initdir() {
1706     # only umount if create_empty_image_rootdir() was called to mount it
1707     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1708         _umount_dir "${initdir:?}"
1709     fi
1710 }
1711
1712 umount_loopback() {
1713     # unmount the loopback device from all places. Otherwise we risk file
1714     # system corruption.
1715     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1716         ddebug "Unmounting all uses of $device"
1717         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1718     done
1719 }
1720
1721 create_empty_image_rootdir() {
1722     create_empty_image
1723     mount_initdir
1724 }
1725
1726 check_asan_reports() {
1727     local ret=0
1728     local root="${1:?}"
1729     local log report
1730
1731     if get_bool "$IS_BUILT_WITH_ASAN"; then
1732         ls -l "$root"
1733         if [[ -e "$root/systemd.asan.log.1" ]]; then
1734             cat "$root/systemd.asan.log.1"
1735             ret=$((ret+1))
1736         fi
1737
1738         for log in pid1 journald; do
1739             report="$(find "$root" -name "systemd-$log.*san.log*" -exec cat {} \;)"
1740             if [[ -n "$report" ]]; then
1741                 printf "%s\n" "$report"
1742                 # shellcheck disable=SC2015
1743                 [[ "$log" == journald ]] && cat "$root/systemd-journald.out" || :
1744                 ret=$((ret+1))
1745             fi
1746         done
1747
1748         # May 08 13:23:31 H testleak[2907148]: SUMMARY: AddressSanitizer: 4 byte(s) leaked in 1 allocation(s).
1749         pids="$(
1750             "$JOURNALCTL" -D "$root/var/log/journal" --grep 'SUMMARY: .*Sanitizer:' |
1751                    grep -v -E 'dbus-daemon|dbus-broker-launch' |
1752                    sed -r -n 's/.* .+\[([0-9]+)\]: SUMMARY:.*/\1/p'
1753         )"
1754
1755         if [[ -n "$pids" ]]; then
1756             ret=$((ret+1))
1757             for pid in $pids; do
1758                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1759             done
1760         fi
1761     fi
1762
1763     return $ret
1764 }
1765
1766 check_coverage_reports() {
1767     local root="${1:?}"
1768
1769     if get_bool "$NO_BUILD"; then
1770         return 0
1771     fi
1772     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1773         return 0
1774     fi
1775
1776     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1777         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1778     else
1779         dest="${TESTDIR:?}/coverage-info"
1780     fi
1781
1782     if [[ ! -e "${TESTDIR:?}/coverage-base" ]]; then
1783         # This shouldn't happen, as the report is generated during the setup
1784         # phase (test_setup()).
1785         derror "Missing base coverage report"
1786         return 1
1787     fi
1788
1789     # Create a coverage report that will later be uploaded. Remove info about system
1790     # libraries/headers and generated files, as we don't really care about them.
1791     lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1792     if [[ -f "$dest" ]]; then
1793         # If the destination report file already exists, don't overwrite it, but
1794         # merge it with the already present one - this usually happens when
1795         # running both "parts" of a test in one run (the qemu and the nspawn part).
1796         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1797     else
1798         # If there's no prior coverage report, merge the new one with the base
1799         # report we did during the setup phase (see test_setup()).
1800         lcov --add-tracefile "${TESTDIR:?}/coverage-base" --add-tracefile "${dest}.new" -o "${dest}"
1801     fi
1802     lcov --remove "$dest" -o "$dest" '/usr/include/*' '/usr/lib/*' "${BUILD_DIR:?}/*"
1803     rm -f "${dest}.new"
1804
1805     # If the test logs contain lines like:
1806     #
1807     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1808     #
1809     # it means we're possibly missing some coverage since gcov can't write the stats,
1810     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1811     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1812     # `setfacl` stuff in install_compiled_systemd().
1813     #
1814     # Also, a note: some tests, like TEST-46, overmount /home with tmpfs, which
1815     # means if your build dir is under /home/your-user (which is usually the
1816     # case) you might get bogus errors and missing coverage.
1817     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1818        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1819         derror "Detected possibly missing coverage, check the journal"
1820         return 1
1821     fi
1822
1823     return 0
1824 }
1825
1826 save_journal() {
1827     local source_dir="${1:?}"
1828     local state="${2:?}"
1829     # Default to always saving journal
1830     local save="yes"
1831     local dest_dir dest_name dest
1832
1833     if [[ "${TEST_SAVE_JOURNAL:-}" == "no" ]]; then
1834         save="no"
1835     elif [[ "${TEST_SAVE_JOURNAL:-}" == "fail" && "$state" -eq 0 ]]; then
1836         save="no"
1837     fi
1838
1839     if [[ -n "${ARTIFACT_DIRECTORY:-}" ]]; then
1840         dest_dir="$ARTIFACT_DIRECTORY"
1841         dest_name="${testname:?}.journal"
1842     else
1843         dest_dir="${TESTDIR:?}"
1844         dest_name="system.journal"
1845     fi
1846
1847     # Show messages from the testsuite-XX.service or messages with priority "warning" and higher
1848     echo " --- $source_dir ---"
1849     "$JOURNALCTL" --all --no-pager --no-hostname -o short-monotonic -D "$source_dir" \
1850         _SYSTEMD_UNIT="testsuite-${TESTID:?}.service" + SYSLOG_IDENTIFIER="testsuite-$TESTID.sh" + \
1851         PRIORITY=4 + PRIORITY=3 + PRIORITY=2 + PRIORITY=1 + PRIORITY=0
1852
1853     if get_bool "$save"; then
1854         # If we don't have systemd-journal-remote copy all journals from /var/log/journal/
1855         # to $dest_dir/journals/ as is, otherwise merge all journals into a single .journal
1856         # file
1857         if [[ -z "${SYSTEMD_JOURNAL_REMOTE:-}" ]]; then
1858             dest="$dest_dir/journals"
1859             mkdir -p "$dest"
1860             cp -a "$source_dir/*" "$dest/"
1861         else
1862             dest="$dest_dir/$dest_name"
1863             "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $source_dir"
1864         fi
1865
1866         if [[ -n "${SUDO_USER:-}" ]]; then
1867             setfacl -R -m "user:$SUDO_USER:r-X" "$dest"
1868         fi
1869
1870         # we want to print this sometime later, so save this in a variable
1871         JOURNAL_LIST="$(ls -lR "$dest")"
1872     fi
1873
1874     rm -rf "${source_dir:?}"/*
1875 }
1876
1877 check_result_common() {
1878     local workspace="${1:?}"
1879     local ret
1880
1881     if [ -s "$workspace/failed" ]; then
1882         # Non-empty …/failed has highest priority
1883         cp -a "$workspace/failed" "${TESTDIR:?}/"
1884         if [ -n "${SUDO_USER}" ]; then
1885             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1886         fi
1887         ret=1
1888     elif get_bool "$TIMED_OUT"; then
1889         echo "(timeout)" >"${TESTDIR:?}/failed"
1890         ret=2
1891     elif [ -e "$workspace/testok" ]; then
1892         # …/testok always counts (but with lower priority than …/failed)
1893         ret=0
1894     elif [ -e "$workspace/skipped" ]; then
1895         # …/skipped always counts (a message is expected)
1896         echo "${TESTNAME:?} was skipped:"
1897         cat "$workspace/skipped"
1898         ret=0
1899     else
1900         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1901         ret=3
1902     fi
1903
1904     check_asan_reports "$workspace" || ret=4
1905
1906     check_coverage_reports "$workspace" || ret=5
1907
1908     save_journal "$workspace/var/log/journal" $ret
1909
1910     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1911         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1912     fi
1913
1914     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1915         echo -n "${TESTNAME:?}: "
1916         cat "$TESTDIR/failed"
1917     fi
1918     echo "${JOURNAL_LIST:-"No journals were saved"}"
1919
1920     return ${ret:?}
1921 }
1922
1923 check_result_nspawn() {
1924     local workspace="${1:?}"
1925     local ret=0
1926
1927     # Run a test-specific checks if defined by check_result_nspawn_hook()
1928     if declare -F check_result_nspawn_hook >/dev/null; then
1929         if ! check_result_nspawn_hook "${workspace}"; then
1930             derror "check_result_nspawn_hook() returned with EC > 0"
1931             ret=4
1932         fi
1933     fi
1934
1935     check_result_common "${workspace}" || ret=$?
1936
1937     _umount_dir "${initdir:?}"
1938
1939     return $ret
1940 }
1941
1942 # can be overridden in specific test
1943 check_result_qemu() {
1944     local ret=0
1945     mount_initdir
1946
1947     # Run a test-specific checks if defined by check_result_qemu_hook()
1948     if declare -F check_result_qemu_hook >/dev/null; then
1949         if ! check_result_qemu_hook "${initdir:?}"; then
1950             derror "check_result_qemu_hook() returned with EC > 0"
1951             ret=4
1952         fi
1953     fi
1954
1955     check_result_common "${initdir:?}" || ret=$?
1956
1957     _umount_dir "${initdir:?}"
1958
1959     return $ret
1960 }
1961
1962 check_result_nspawn_unittests() {
1963     local workspace="${1:?}"
1964     local ret=1
1965
1966     [[ -e "$workspace/testok" ]] && ret=0
1967
1968     if [[ -s "$workspace/failed" ]]; then
1969         ret=$((ret + 1))
1970         echo "=== Failed test log ==="
1971         cat "$workspace/failed"
1972     else
1973         if [[ -s "$workspace/skipped" ]]; then
1974             echo "=== Skipped test log =="
1975             cat "$workspace/skipped"
1976             # We might have only skipped tests - that should not fail the job
1977             ret=0
1978         fi
1979         if [[ -s "$workspace/testok" ]]; then
1980             echo "=== Passed tests ==="
1981             cat "$workspace/testok"
1982         fi
1983     fi
1984
1985     get_bool "${TIMED_OUT:=}" && ret=1
1986     check_coverage_reports "$workspace" || ret=5
1987
1988     save_journal "$workspace/var/log/journal" $ret
1989     echo "${JOURNAL_LIST:-"No journals were saved"}"
1990
1991     _umount_dir "${initdir:?}"
1992
1993     return $ret
1994 }
1995
1996 check_result_qemu_unittests() {
1997     local ret=1
1998
1999     mount_initdir
2000     [[ -e "${initdir:?}/testok" ]] && ret=0
2001
2002     if [[ -s "$initdir/failed" ]]; then
2003         ret=$((ret + 1))
2004         echo "=== Failed test log ==="
2005         cat "$initdir/failed"
2006     else
2007         if [[ -s "$initdir/skipped" ]]; then
2008             echo "=== Skipped test log =="
2009             cat "$initdir/skipped"
2010             # We might have only skipped tests - that should not fail the job
2011             ret=0
2012         fi
2013         if [[ -s "$initdir/testok" ]]; then
2014             echo "=== Passed tests ==="
2015             cat "$initdir/testok"
2016         fi
2017     fi
2018
2019     get_bool "${TIMED_OUT:=}" && ret=1
2020     check_coverage_reports "$initdir" || ret=5
2021
2022     save_journal "$initdir/var/log/journal" $ret
2023     echo "${JOURNAL_LIST:-"No journals were saved"}"
2024
2025     _umount_dir "$initdir"
2026
2027     return $ret
2028 }
2029
2030 create_rc_local() {
2031     dinfo "Create rc.local"
2032     mkdir -p "${initdir:?}/etc/rc.d"
2033     cat >"$initdir/etc/rc.d/rc.local" <<EOF
2034 #!/usr/bin/env bash
2035 exit 0
2036 EOF
2037     chmod 0755 "$initdir/etc/rc.d/rc.local"
2038 }
2039
2040 install_execs() {
2041     ddebug "Install executables from the service files"
2042
2043     local pkg_config_path="${BUILD_DIR:?}/src/core/"
2044     local systemunitdir userunitdir exe
2045     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
2046     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
2047     while read -r exe; do
2048         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
2049         # also, plymouth is pulled in by rescue.service, but even there the exit code
2050         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
2051         dinfo "Attempting to install $exe (based on unit file reference)"
2052         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
2053     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
2054 }
2055
2056 generate_module_dependencies() {
2057     dinfo "Generate modules dependencies"
2058     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
2059         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
2060             dfatal "\"depmod -a $KERNEL_VER\" failed."
2061             exit 1
2062     fi
2063 }
2064
2065 install_depmod_files() {
2066     dinfo "Install depmod files"
2067     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
2068     inst "/lib/modules/$KERNEL_VER/modules.builtin"
2069 }
2070
2071 install_plymouth() {
2072     dinfo "Install plymouth"
2073     # install plymouth, if found... else remove plymouth service files
2074     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
2075     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
2076     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
2077     #         image_install plymouth plymouthd
2078     # else
2079         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
2080     # fi
2081 }
2082
2083 install_haveged() {
2084     # If haveged is installed, it's probably included in initrd and needs to be
2085     # installed in the image too.
2086     if [ -x /usr/sbin/haveged ]; then
2087         dinfo "Install haveged files"
2088         inst /usr/sbin/haveged
2089         for u in /usr/lib/systemd/system/haveged*; do
2090             inst "$u"
2091         done
2092     fi
2093 }
2094
2095 install_ld_so_conf() {
2096     dinfo "Install /etc/ld.so.conf*"
2097     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
2098     ldconfig -r "$initdir"
2099 }
2100
2101 install_testuser() {
2102     dinfo "Set up a test user"
2103     # create unprivileged user for user manager tests
2104     mkdir -p "${initdir:?}/etc/sysusers.d"
2105     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
2106 u testuser    4711     "Test User" /home/testuser /bin/bash
2107 EOF
2108
2109     mkdir -p "$initdir/home/testuser"
2110     chmod 0700 "$initdir/home/testuser"
2111     chown 4711:4711 "$initdir/home/testuser"
2112 }
2113
2114 install_config_files() {
2115     dinfo "Install config files"
2116     inst /etc/sysconfig/init || :
2117     inst /etc/passwd
2118     inst /etc/shadow
2119     inst_any /etc/login.defs /usr/etc/login.defs
2120     inst /etc/group
2121     inst /etc/shells
2122     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
2123     inst /etc/pam.conf || :
2124     inst_any /etc/os-release /usr/lib/os-release
2125     inst /etc/localtime
2126     # we want an empty environment
2127     : >"${initdir:?}/etc/environment"
2128     : >"$initdir/etc/machine-id"
2129     : >"$initdir/etc/resolv.conf"
2130
2131     # set the hostname
2132     echo 'H' >"$initdir/etc/hostname"
2133
2134     # let's set up just one image with the traditional verbose output
2135     if [ "${IMAGE_NAME:?}" != "basic" ]; then
2136         mkdir -p "$initdir/etc/systemd/system.conf.d"
2137         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
2138     fi
2139 }
2140
2141 install_basic_tools() {
2142     dinfo "Install basic tools"
2143     image_install "${BASICTOOLS[@]}"
2144     image_install -o sushell
2145     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
2146     image_install -o ldconfig.real
2147 }
2148
2149 install_debug_tools() {
2150     dinfo "Install debug tools"
2151     image_install -o "${DEBUGTOOLS[@]}"
2152
2153     if get_bool "$INTERACTIVE_DEBUG"; then
2154         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
2155         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
2156         mkdir -p "$getty_override"
2157         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
2158         echo 'export TERM=linux' >>"$initdir/etc/profile"
2159
2160         if command -v resize >/dev/null; then
2161             image_install resize
2162             echo "resize" >>"$initdir/etc/profile"
2163         fi
2164
2165         # Sometimes we might end up with plymouthd still running (especially
2166         # with the initrd -> asan_wrapper -> systemd transition), which will eat
2167         # our inputs and make debugging via tty impossible. Let's fix this by
2168         # killing plymouthd explicitly for the interactive sessions.
2169         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
2170         #   - https://github.com/llvm/llvm-project/issues/49223
2171         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
2172         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
2173         cat >"$plymouth_unit" <<EOF
2174 [Unit]
2175 After=multi-user.target
2176
2177 [Service]
2178 ExecStart=sh -c 'killall --verbose plymouthd || :'
2179
2180 [Install]
2181 WantedBy=multi-user.target
2182 EOF
2183         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
2184     fi
2185 }
2186
2187 install_libnss() {
2188     dinfo "Install libnss"
2189     # install libnss_files for login
2190     local NSS_LIBS
2191     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
2192     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
2193         image_install "${NSS_LIBS[@]}"
2194     fi
2195 }
2196
2197 install_dbus() {
2198     dinfo "Install dbus"
2199     inst "${ROOTLIBDIR:?}/system/dbus.socket"
2200
2201     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2202     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
2203         inst "$ROOTLIBDIR/system/dbus-broker.service"
2204         inst /usr/bin/dbus-broker
2205         inst /usr/bin/dbus-broker-launch
2206         image_install -o {/etc,/usr/lib}/systemd/system/dbus.service
2207     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
2208         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2209         inst "$ROOTLIBDIR/system/dbus-daemon.service"
2210         # Alias symlink
2211         image_install -o {/etc,/usr/lib}/systemd/system/dbus.service
2212     else
2213         inst "$ROOTLIBDIR/system/dbus.service"
2214     fi
2215
2216     while read -r file; do
2217         inst "$file"
2218     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
2219
2220     # setup policy for Type=dbus test
2221     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
2222     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
2223 <?xml version="1.0"?>
2224 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2225         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2226 <busconfig>
2227     <policy user="root">
2228         <allow own="systemd.test.ExecStopPost"/>
2229     </policy>
2230 </busconfig>
2231 EOF
2232
2233     # If we run without KVM, bump the service start timeout
2234     if ! get_bool "$QEMU_KVM"; then
2235         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
2236 <?xml version="1.0"?>
2237 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2238         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2239 <busconfig>
2240     <limit name="service_start_timeout">120000</limit>
2241 </busconfig>
2242 EOF
2243         # Bump the client-side timeout in sd-bus as well
2244         mkdir -p "$initdir/etc/systemd/system.conf.d"
2245         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=120' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
2246     fi
2247 }
2248
2249 install_user_dbus() {
2250     dinfo "Install user dbus"
2251     local userunitdir
2252     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
2253         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
2254         userunitdir=/usr/lib/systemd/user
2255     fi
2256
2257     inst "$userunitdir/dbus.socket"
2258     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
2259
2260     # Append the After= dependency on dbus in case it isn't already set up
2261     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
2262     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
2263 [Unit]
2264 After=dbus.service
2265 EOF
2266
2267     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2268     if [ -f "$userunitdir/dbus-broker.service" ]; then
2269         inst "$userunitdir/dbus-broker.service"
2270         image_install -o {/etc,/usr/lib}/systemd/user/dbus.service
2271     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
2272         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2273         inst "$userunitdir/dbus-daemon.service"
2274         # Alias symlink
2275         image_install -o {/etc,/usr/lib}/systemd/user/dbus.service
2276     else
2277         inst "$userunitdir/dbus.service"
2278     fi
2279 }
2280
2281 install_pam() {
2282     dinfo "Install PAM"
2283     local paths=()
2284
2285     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
2286         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
2287     else
2288         paths+=(/lib*/security)
2289     fi
2290
2291     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
2292         [ -d "$d" ] && paths+=("$d")
2293     done
2294
2295     while read -r file; do
2296         inst "$file"
2297     done < <(find "${paths[@]}" -xtype f)
2298
2299     # pam_unix depends on unix_chkpwd.
2300     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2301     image_install -o unix_chkpwd
2302
2303     # set empty root password for easy debugging
2304     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2305
2306     # And make sure pam_unix will accept it by making sure that
2307     # the PAM module has the nullok option.
2308     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2309         [ -d "$initdir/$d" ] || continue
2310         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2311     done
2312 }
2313
2314 install_locales() {
2315     # install only C.UTF-8 and English locales
2316     dinfo "Install locales"
2317
2318     if command -v meson >/dev/null \
2319             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2320             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2321         # locale-gen support
2322         image_install -o locale-gen localedef
2323         inst /etc/locale.gen || :
2324         inst /usr/share/i18n/SUPPORTED || :
2325         inst_recursive /usr/share/i18n/charmaps
2326         inst_recursive /usr/share/i18n/locales
2327         inst_recursive /usr/share/locale/en*
2328         inst_recursive /usr/share/locale/de*
2329         image_install /usr/share/locale/locale.alias
2330         # locale-gen might either generate each locale separately or merge them
2331         # into a single archive
2332         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2333               image_install /usr/lib/locale/locale-archive); then
2334             dfatal "Failed to install required locales"
2335             exit 1
2336         fi
2337     else
2338         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2339     fi
2340 }
2341
2342 # shellcheck disable=SC2120
2343 install_keymaps() {
2344     local i p
2345     local -a prefix=(
2346         "/usr/lib"
2347         "/usr/share"
2348     )
2349
2350     dinfo "Install console keymaps"
2351
2352     if (( $# == 0 )); then
2353         for p in "${prefix[@]}"; do
2354             # The first three paths may be deprecated.
2355             # It seems now the last three paths are used by many distributions.
2356             for i in \
2357                 "$p"/kbd/keymaps/include/* \
2358                 "$p"/kbd/keymaps/i386/include/* \
2359                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2360                 "$p"/kbd/keymaps/legacy/include/* \
2361                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2362                 "$p"/kbd/keymaps/xkb/us*; do
2363                     [[ -f "$i" ]] || continue
2364                     inst "$i"
2365             done
2366         done
2367     else
2368         # When it takes any argument, then install more keymaps.
2369         for p in "${prefix[@]}"; do
2370             for i in \
2371                 "$p"/kbd/keymaps/include/* \
2372                 "$p"/kbd/keymaps/i386/*/* \
2373                 "$p"/kbd/keymaps/legacy/i386/*/* \
2374                 "$p"/kbd/keymaps/xkb/*; do
2375                     [[ -f "$i" ]] || continue
2376                     inst "$i"
2377             done
2378         done
2379     fi
2380 }
2381
2382 install_x11_keymaps() {
2383     dinfo "Install x11 keymaps"
2384
2385     if (( $# == 0 )); then
2386         # Install only keymap list.
2387         inst /usr/share/X11/xkb/rules/base.lst
2388     else
2389         # When it takes any argument, then install all keymaps.
2390         inst_recursive /usr/share/X11/xkb
2391     fi
2392 }
2393
2394 install_zoneinfo() {
2395     dinfo "Install time zones"
2396     inst_any /usr/share/zoneinfo/Asia/Seoul
2397     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2398     inst_any /usr/share/zoneinfo/Australia/Sydney
2399     inst_any /usr/share/zoneinfo/Europe/Berlin
2400     inst_any /usr/share/zoneinfo/Europe/Dublin
2401     inst_any /usr/share/zoneinfo/Europe/Kiev
2402     inst_any /usr/share/zoneinfo/Pacific/Auckland
2403     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2404     inst_any /usr/share/zoneinfo/CET
2405     inst_any /usr/share/zoneinfo/EET
2406     inst_any /usr/share/zoneinfo/UTC
2407 }
2408
2409 install_fonts() {
2410     dinfo "Install system fonts"
2411     for i in \
2412         /usr/{lib,share}/kbd/consolefonts/eurlatgr* \
2413         /usr/{lib,share}/kbd/consolefonts/latarcyrheb-sun16*; do
2414             [[ -f "$i" ]] || continue
2415             inst "$i"
2416     done
2417 }
2418
2419 install_terminfo() {
2420     dinfo "Install terminfo files"
2421     local terminfodir
2422     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2423         [ -f "${terminfodir}/l/linux" ] && break
2424     done
2425     image_install -o "${terminfodir}/l/linux"
2426 }
2427
2428 has_user_dbus_socket() {
2429     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2430         return 0
2431     else
2432         echo "Per-user instances are not supported. Skipping..."
2433         return 1
2434     fi
2435 }
2436
2437 setup_nspawn_root_hook() { :;}
2438
2439 setup_nspawn_root() {
2440     if [ -z "${initdir}" ]; then
2441         dfatal "\$initdir not defined"
2442         exit 1
2443     fi
2444
2445     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2446
2447     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2448         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2449         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2450     fi
2451
2452     setup_nspawn_root_hook
2453 }
2454
2455 setup_basic_dirs() {
2456     mkdir -p "${initdir:?}/run"
2457     mkdir -p "$initdir/etc/systemd/system"
2458     mkdir -p "$initdir/var/log/journal"
2459
2460
2461     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2462         if [ -L "/$d" ]; then
2463             inst_symlink "/$d"
2464         else
2465             inst_dir "/$d"
2466         fi
2467     done
2468
2469     ln -sfn /run "$initdir/var/run"
2470     ln -sfn /run/lock "$initdir/var/lock"
2471 }
2472
2473 mask_supporting_services() {
2474     # mask some services that we do not want to run in these tests
2475     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2476     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2477     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2478     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2479     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2480 }
2481
2482 inst_libs() {
2483     local bin="${1:?}"
2484     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2485     local file line
2486
2487     while read -r line; do
2488         [[ "$line" = 'not a dynamic executable' ]] && break
2489         # Ignore errors about our own stuff missing. This is most likely caused
2490         # by ldd attempting to use the unprefixed RPATH.
2491         [[ "$line" =~ (libsystemd|libudev).*\ not\ found ]] && continue
2492
2493         if [[ "$line" =~ not\ found ]]; then
2494             dfatal "Missing a shared library required by $bin."
2495             dfatal "Run \"ldd $bin\" to find out what it is."
2496             dfatal "$line"
2497             dfatal "Cannot create a test image."
2498             exit 1
2499         fi
2500
2501         if [[ "$line" =~ $so_regex ]]; then
2502             file="${BASH_REMATCH[1]}"
2503             [[ -e "${initdir:?}/$file" ]] && continue
2504             inst_library "$file"
2505         fi
2506     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2507 }
2508
2509 import_testdir() {
2510     # make sure we don't get a stale LOOPDEV value from old times
2511     local _LOOPDEV="${LOOPDEV:=}"
2512     # We don't want shellcheck to follow & check the $STATEFILE
2513     # shellcheck source=/dev/null
2514     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2515     LOOPDEV="$_LOOPDEV"
2516     if [[ ! -d "$TESTDIR" ]]; then
2517         if [[ -z "$TESTDIR" ]]; then
2518             TESTDIR="$(mktemp --tmpdir=$WORKDIR -d -t systemd-test.XXXXXX)"
2519         else
2520             mkdir -p "$TESTDIR"
2521         fi
2522
2523         cat >"$STATEFILE" <<EOF
2524 TESTDIR="$TESTDIR"
2525 EOF
2526         export TESTDIR
2527     fi
2528
2529     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2530     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2531 }
2532
2533 import_initdir() {
2534     initdir="${TESTDIR:?}/root"
2535     mkdir -p "$initdir"
2536     export initdir
2537 }
2538
2539 get_cgroup_hierarchy() {
2540     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2541         cgroup2fs)
2542             echo "unified"
2543             ;;
2544         tmpfs)
2545             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2546                 echo "hybrid"
2547             else
2548                 echo "legacy"
2549             fi
2550             ;;
2551         *)
2552             dfatal "Failed to determine host's cgroup hierarchy"
2553             exit 1
2554     esac
2555 }
2556
2557 ## @brief Converts numeric logging level to the first letter of level name.
2558 #
2559 # @param lvl Numeric logging level in range from 1 to 6.
2560 # @retval 1 if @a lvl is out of range.
2561 # @retval 0 if @a lvl is correct.
2562 # @result Echoes first letter of level name.
2563 _lvl2char() {
2564     case "$1" in
2565         1) echo F;;
2566         2) echo E;;
2567         3) echo W;;
2568         4) echo I;;
2569         5) echo D;;
2570         6) echo T;;
2571         *) return 1;;
2572     esac
2573 }
2574
2575 ## @brief Internal helper function for _do_dlog()
2576 #
2577 # @param lvl Numeric logging level.
2578 # @param msg Message.
2579 # @retval 0 It's always returned, even if logging failed.
2580 #
2581 # @note This function is not supposed to be called manually. Please use
2582 # dtrace(), ddebug(), or others instead which wrap this one.
2583 #
2584 # This function calls _do_dlog() either with parameter msg, or if
2585 # none is given, it will read standard input and will use every line as
2586 # a message.
2587 #
2588 # This enables:
2589 # dwarn "This is a warning"
2590 # echo "This is a warning" | dwarn
2591 LOG_LEVEL="${LOG_LEVEL:-4}"
2592
2593 dlog() {
2594     local lvl lvlc
2595
2596     [ -z "$LOG_LEVEL" ] && return 0
2597     lvl="${1:?}"; shift
2598     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2599     lvlc="$(_lvl2char "$lvl")" || return 0
2600
2601     if [ $# -ge 1 ]; then
2602         echo "$lvlc: $*"
2603     else
2604         while read -r line; do
2605             echo "$lvlc: " "$line"
2606         done
2607     fi
2608 }
2609
2610 ## @brief Logs message at TRACE level (6)
2611 #
2612 # @param msg Message.
2613 # @retval 0 It's always returned, even if logging failed.
2614 dtrace() {
2615     set +x
2616     dlog 6 "$@"
2617     if get_bool "${debug:=}"; then
2618         set -x
2619     fi
2620 }
2621
2622 ## @brief Logs message at DEBUG level (5)
2623 #
2624 # @param msg Message.
2625 # @retval 0 It's always returned, even if logging failed.
2626 ddebug() {
2627     dlog 5 "$@"
2628 }
2629
2630 ## @brief Logs message at INFO level (4)
2631 #
2632 # @param msg Message.
2633 # @retval 0 It's always returned, even if logging failed.
2634 dinfo() {
2635     set +x
2636     dlog 4 "$@"
2637     if get_bool "${debug:=}"; then
2638         set -x
2639     fi
2640 }
2641
2642 ## @brief Logs message at WARN level (3)
2643 #
2644 # @param msg Message.
2645 # @retval 0 It's always returned, even if logging failed.
2646 dwarn() {
2647     set +x
2648     dlog 3 "$@"
2649     if get_bool "${debug:=}"; then
2650         set -x
2651     fi
2652 }
2653
2654 ## @brief Logs message at ERROR level (2)
2655 #
2656 # @param msg Message.
2657 # @retval 0 It's always returned, even if logging failed.
2658 derror() {
2659     dlog 2 "$@"
2660 }
2661
2662 ## @brief Logs message at FATAL level (1)
2663 #
2664 # @param msg Message.
2665 # @retval 0 It's always returned, even if logging failed.
2666 dfatal() {
2667     set +x
2668     dlog 1 "$@"
2669     if get_bool "${debug:=}"; then
2670         set -x
2671     fi
2672 }
2673
2674
2675 # Generic substring function.  If $2 is in $1, return 0.
2676 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2677
2678 # normalize_path <path>
2679 # Prints the normalized path, where it removes any duplicated
2680 # and trailing slashes.
2681 # Example:
2682 # $ normalize_path ///test/test//
2683 # /test/test
2684 normalize_path() {
2685     shopt -q -s extglob
2686     set -- "${1//+(\/)//}"
2687     shopt -q -u extglob
2688     echo "${1%/}"
2689 }
2690
2691 # convert_abs_rel <from> <to>
2692 # Prints the relative path, when creating a symlink to <to> from <from>.
2693 # Example:
2694 # $ convert_abs_rel /usr/bin/test /bin/test-2
2695 # ../../bin/test-2
2696 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2697 convert_abs_rel() {
2698     local __current __absolute __abssize __cursize __newpath
2699     local -i __i __level
2700
2701     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2702
2703     # corner case #1 - self looping link
2704     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2705
2706     # corner case #2 - own dir link
2707     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2708
2709     IFS="/" read -ra __current <<< "$1"
2710     IFS="/" read -ra __absolute <<< "$2"
2711
2712     __abssize=${#__absolute[@]}
2713     __cursize=${#__current[@]}
2714
2715     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2716         (( __level++ ))
2717         if (( __level > __abssize || __level > __cursize ))
2718         then
2719             break
2720         fi
2721     done
2722
2723     for ((__i = __level; __i < __cursize-1; __i++)); do
2724         if ((__i > __level))
2725         then
2726             __newpath=$__newpath"/"
2727         fi
2728         __newpath=$__newpath".."
2729     done
2730
2731     for ((__i = __level; __i < __abssize; __i++)); do
2732         if [[ -n $__newpath ]]
2733         then
2734             __newpath=$__newpath"/"
2735         fi
2736         __newpath=$__newpath${__absolute[__i]}
2737     done
2738
2739     echo "$__newpath"
2740 }
2741
2742
2743 # Install a directory, keeping symlinks as on the original system.
2744 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2745 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2746 # and a symlink ${initdir}/lib -> lib64.
2747 inst_dir() {
2748     local dir="${1:?}"
2749     local part="${dir%/*}"
2750     local file
2751
2752     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2753
2754     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2755         dir="$part $dir"
2756         part="${part%/*}"
2757     done
2758
2759     # iterate over parent directories
2760     for file in $dir; do
2761         [[ -e "${initdir}/$file" ]] && continue
2762         if [[ -L $file ]]; then
2763             inst_symlink "$file"
2764         else
2765             # create directory
2766             mkdir -m 0755 "${initdir}/$file" || return 1
2767             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2768             chmod u+w "${initdir}/$file"
2769         fi
2770     done
2771 }
2772
2773 # $1 = file to copy to ramdisk
2774 # $2 (optional) Name for the file on the ramdisk
2775 # Location of the image dir is assumed to be $initdir
2776 # We never overwrite the target if it exists.
2777 inst_simple() {
2778     [[ -f "${1:?}" ]] || return 1
2779     strstr "$1" "/" || return 1
2780
2781     local src="$1"
2782     local target="${2:-$1}"
2783     if ! [[ -d ${initdir:?}/$target ]]; then
2784         [[ -e ${initdir}/$target ]] && return 0
2785         [[ -L ${initdir}/$target ]] && return 0
2786         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2787     fi
2788     # install checksum files also
2789     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2790         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2791     fi
2792     ddebug "Installing $src"
2793     cp --sparse=always --force --dereference --preserve=all "$src" "${initdir}/$target"
2794 }
2795
2796 # find symlinks linked to given library file
2797 # $1 = library file
2798 # Function searches for symlinks by stripping version numbers appended to
2799 # library filename, checks if it points to the same target and finally
2800 # prints the list of symlinks to stdout.
2801 #
2802 # Example:
2803 # rev_lib_symlinks libfoo.so.8.1
2804 # output: libfoo.so.8 libfoo.so
2805 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2806 rev_lib_symlinks() {
2807     local fn="${1:?}"
2808     local links=""
2809     local orig
2810     orig="$(readlink -f "$1")"
2811
2812     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2813
2814     until [[ "${fn##*.}" == so ]]; do
2815         fn="${fn%.*}"
2816         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2817     done
2818
2819     echo "${links}"
2820 }
2821
2822 # Same as above, but specialized to handle dynamic libraries.
2823 # It handles making symlinks according to how the original library
2824 # is referenced.
2825 inst_library() {
2826     local src="${1:?}"
2827     local dest="${2:-$1}"
2828     local reallib symlink
2829
2830     strstr "$1" "/" || return 1
2831     [[ -e ${initdir:?}/$dest ]] && return 0
2832     if [[ -L $src ]]; then
2833         # install checksum files also
2834         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2835             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2836         fi
2837         reallib="$(readlink -f "$src")"
2838         inst_simple "$reallib" "$reallib"
2839         inst_dir "${dest%/*}"
2840         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2841         ddebug "Creating symlink $reallib -> $dest"
2842         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2843     else
2844         inst_simple "$src" "$dest"
2845     fi
2846
2847     # Create additional symlinks.  See rev_symlinks description.
2848     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2849         if [[ ! -e "$initdir/$symlink" ]]; then
2850             ddebug "Creating extra symlink: $symlink"
2851             inst_symlink "$symlink"
2852         fi
2853     done
2854 }
2855
2856 # find a binary.  If we were not passed the full path directly,
2857 # search in the usual places to find the binary.
2858 find_binary() {
2859     local bin="${1:?}"
2860     if [[ -z ${bin##/*} ]]; then
2861         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2862             echo "$bin"
2863             return 0
2864         fi
2865     fi
2866
2867     type -P "$bin"
2868 }
2869
2870 # Same as above, but specialized to install binary executables.
2871 # Install binary executable, and all shared library dependencies, if any.
2872 inst_binary() {
2873     local bin="${1:?}"
2874     local path target
2875
2876     # In certain cases we might attempt to install a binary which is already
2877     # present in the test image, yet it's missing from the host system.
2878     # In such cases, let's check if the binary indeed exists in the image
2879     # before doing any other checks. If it does, immediately return with
2880     # success.
2881     if [[ $# -eq 1 ]]; then
2882         for path in "" bin sbin usr/bin usr/sbin; do
2883             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2884         done
2885     fi
2886
2887     bin="$(find_binary "$bin")" || return 1
2888     target="${2:-$bin}"
2889     [[ -e "${initdir:?}/$target" ]] && return 0
2890     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2891
2892     local file line
2893     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2894     # DSOs provided by systemd
2895     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2896     local wrap_binary=0
2897     local enable_lsan=0
2898     # I love bash!
2899     while read -r line; do
2900         [[ "$line" = 'not a dynamic executable' ]] && break
2901
2902         # Ignore errors about our own stuff missing. This is most likely caused
2903         # by ldd attempting to use the unprefixed RPATH.
2904         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2905
2906         # We're built with ASan and the target binary loads one of the systemd's
2907         # DSOs, so we need to tweak the environment before executing the binary
2908         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2909             wrap_binary=1
2910         fi
2911
2912         if [[ "$line" =~ $so_regex ]]; then
2913             file="${BASH_REMATCH[1]}"
2914             [[ -e "${initdir}/$file" ]] && continue
2915             inst_library "$file"
2916             continue
2917         fi
2918
2919         if [[ "$line" =~ not\ found ]]; then
2920             dfatal "Missing a shared library required by $bin."
2921             dfatal "Run \"ldd $bin\" to find out what it is."
2922             dfatal "$line"
2923             dfatal "Cannot create a test image."
2924             exit 1
2925         fi
2926     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2927
2928     # Same as above, but we need to wrap certain libraries unconditionally
2929     #
2930     # chgrp, chown, getent, login, setfacl, su, useradd, userdel
2931     #    - dlopen() (not only) systemd's PAM modules
2932     # ls, mkfs.*, mksquashfs, mkswap, setpriv, stat
2933     #   - pull in nss_systemd with certain options (like ls -l) when
2934     #     nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2935     # delv, dig - pull in nss_resolve if `resolve` is in nsswitch.conf
2936     # tar - called by machinectl in TEST-25
2937     bin_rx='/(agetty|chgrp|chown|curl|delv|dig|getfacl|getent|id|login|ls|mkfs\.[a-z0-9]+|mksquashfs|mkswap|setfacl|setpriv|stat|su|tar|useradd|userdel)$'
2938     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ $bin_rx ]]; then
2939         wrap_binary=1
2940         # Ugh, so we want to disable LSan in most cases for the wrapped binaries, since
2941         # we don't care about memory leaks in such binaries. However, in certain cases
2942         # the external binary is the only interface for the systemd code, like for
2943         # the systemd NSS modules, where we want to detect memory leaks. So let's
2944         # do another check to decide if we want to enable LSan for given binary.
2945         if [[ "$bin" =~ /getent$ ]]; then
2946             enable_lsan=1
2947         fi
2948     fi
2949
2950     # If the target binary is built with ASan support, we don't need to wrap
2951     # it, as it should handle everything by itself
2952     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2953         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2954         # Install the target binary with a ".orig" suffix
2955         inst_simple "$bin" "${target}.orig"
2956         # Create a simple shell wrapper in place of the target binary, which
2957         # sets necessary ASan-related env variables and then exec()s the
2958         # suffixed target binary
2959         cat >"$initdir/$target" <<EOF
2960 #!/bin/bash
2961 # Preload the ASan runtime DSO, otherwise ASAn will complain
2962 export LD_PRELOAD="$ASAN_RT_PATH"
2963 # Disable LSan to speed things up, since we don't care about leak reports
2964 # from 'external' binaries
2965 export ASAN_OPTIONS=detect_leaks=$enable_lsan
2966 # Set argv[0] to the original binary name without the ".orig" suffix
2967 exec -a "\$0" -- "${target}.orig" "\$@"
2968 EOF
2969         chmod +x "$initdir/$target"
2970     else
2971         inst_simple "$bin" "$target"
2972     fi
2973 }
2974
2975 # same as above, except for shell scripts.
2976 # If your shell script does not start with shebang, it is not a shell script.
2977 inst_script() {
2978     local bin line shebang_regex
2979     bin="$(find_binary "${1:?}")" || return 1
2980     shift
2981
2982     read -r -n 80 line <"$bin"
2983     # If debug is set, clean unprintable chars to prevent messing up the term
2984     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2985     shebang_regex='(#! *)(/[^ ]+).*'
2986     [[ "$line" =~ $shebang_regex ]] || return 1
2987     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2988 }
2989
2990 # same as above, but specialized for symlinks
2991 inst_symlink() {
2992     local src="${1:?}"
2993     local target="${2:-$src}"
2994     local realsrc
2995
2996     strstr "$src" "/" || return 1
2997     [[ -L "$src" ]] || return 1
2998     [[ -L "${initdir:?}/$target" ]] && return 0
2999     realsrc="$(readlink -f "$src")"
3000     if ! [[ -e "$initdir/$realsrc" ]]; then
3001         if [[ -d "$realsrc" ]]; then
3002             inst_dir "$realsrc"
3003         else
3004             inst "$realsrc"
3005         fi
3006     fi
3007     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
3008     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
3009     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
3010 }
3011
3012 # attempt to install any programs specified in a udev rule
3013 inst_rule_programs() {
3014     local rule="${1:?}"
3015     local prog bin
3016
3017     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
3018         if [ -x "/lib/udev/$prog" ]; then
3019             bin="/lib/udev/$prog"
3020         else
3021             if ! bin="$(find_binary "$prog")"; then
3022                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
3023                 continue
3024             fi
3025         fi
3026
3027         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
3028         image_install "$bin"
3029     done
3030 }
3031
3032 # udev rules always get installed in the same place, so
3033 # create a function to install them to make life simpler.
3034 inst_rules() {
3035     local target=/etc/udev/rules.d
3036     local found rule
3037
3038     inst_dir "/lib/udev/rules.d"
3039     inst_dir "$target"
3040     for rule in "$@"; do
3041         if [ "${rule#/}" = "$rule" ]; then
3042             for r in /lib/udev/rules.d /etc/udev/rules.d; do
3043                 if [[ -f "$r/$rule" ]]; then
3044                     found="$r/$rule"
3045                     inst_simple "$found"
3046                     inst_rule_programs "$found"
3047                 fi
3048             done
3049         fi
3050         for r in '' ./; do
3051             if [[ -f "${r}${rule}" ]]; then
3052                 found="${r}${rule}"
3053                 inst_simple "$found" "$target/${found##*/}"
3054                 inst_rule_programs "$found"
3055             fi
3056         done
3057         [[ $found ]] || dinfo "Skipping udev rule: $rule"
3058         found=
3059     done
3060 }
3061
3062 # general purpose installation function
3063 # Same args as above.
3064 inst() {
3065     case $# in
3066         1) ;;
3067         2)
3068             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
3069             [[ "$initdir" = "$2" ]] && set "$1"
3070             ;;
3071         3)
3072             [[ -z "$initdir" ]] && export initdir="$2"
3073             set "$1" "$3"
3074             ;;
3075         *)
3076             dfatal "inst only takes 1 or 2 or 3 arguments"
3077             exit 1
3078             ;;
3079     esac
3080
3081     local fun
3082     for fun in inst_symlink inst_script inst_binary inst_simple; do
3083         "$fun" "$@" && return 0
3084     done
3085
3086     dwarn "Failed to install '$1'"
3087     return 1
3088 }
3089
3090 # install any of listed files
3091 #
3092 # If first argument is '-d' and second some destination path, first accessible
3093 # source is installed into this path, otherwise it will installed in the same
3094 # path as source.  If none of listed files was installed, function return 1.
3095 # On first successful installation it returns with 0 status.
3096 #
3097 # Example:
3098 #
3099 # inst_any -d /bin/foo /bin/bar /bin/baz
3100 #
3101 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
3102 # initrd.
3103 inst_any() {
3104     local dest file
3105
3106     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
3107
3108     for file in "$@"; do
3109         if [[ -e "$file" ]]; then
3110             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
3111             inst "$file" && return 0
3112         fi
3113     done
3114
3115     return 1
3116 }
3117
3118 inst_recursive() {
3119     local p item
3120
3121     for p in "$@"; do
3122         # Make sure the source exists, as the process substitution below
3123         # suppresses errors
3124         stat "$p" >/dev/null || return 1
3125
3126         while read -r item; do
3127             if [[ -d "$item" ]]; then
3128                 inst_dir "$item"
3129             elif [[ -f "$item" ]]; then
3130                 inst_simple "$item"
3131             fi
3132         done < <(find "$p" 2>/dev/null)
3133     done
3134 }
3135
3136 # image_install [-o ] <file> [<file> ... ]
3137 # Install <file> to the test image
3138 # -o optionally install the <file> and don't fail, if it is not there
3139 image_install() {
3140     local optional=no
3141     local prog="${1:?}"
3142
3143     if [[ "$prog" = '-o' ]]; then
3144         optional=yes
3145         shift
3146     fi
3147
3148     for prog in "$@"; do
3149         if ! inst "$prog" ; then
3150             if get_bool "$optional"; then
3151                 dinfo "Skipping program $prog as it cannot be found and is" \
3152                     "flagged to be optional"
3153             else
3154                 dfatal "Failed to install $prog"
3155                 exit 1
3156             fi
3157         fi
3158     done
3159 }
3160
3161 # Install a single kernel module along with any firmware it may require.
3162 # $1 = full path to kernel module to install
3163 install_kmod_with_fw() {
3164     local module="${1:?}"
3165     # no need to go further if the module is already installed
3166     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
3167     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
3168
3169     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
3170
3171     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
3172
3173     local modname="${module##*/}"
3174     local fwdir found fw
3175     modname="${modname%.ko*}"
3176
3177     while read -r fw; do
3178         found=
3179         for fwdir in /lib/firmware/updates /lib/firmware; do
3180             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
3181                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
3182                 found=yes
3183             fi
3184         done
3185         if ! get_bool "$found"; then
3186             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
3187                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
3188                     "\"${modname}.ko\""
3189             else
3190                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
3191                     "\"${modname}.ko\""
3192             fi
3193         fi
3194     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
3195     return 0
3196 }
3197
3198 # Do something with all the dependencies of a kernel module.
3199 # Note that kernel modules depend on themselves using the technique we use
3200 # $1 = function to call for each dependency we find
3201 #      It will be passed the full path to the found kernel module
3202 # $2 = module to get dependencies for
3203 # rest of args = arguments to modprobe
3204 for_each_kmod_dep() {
3205     local func="${1:?}"
3206     local kmod="${2:?}"
3207     local found=0
3208     local cmd modpath
3209     shift 2
3210
3211     while read -r cmd modpath _; do
3212         [[ "$cmd" = insmod ]] || continue
3213         "$func" "$modpath" || return $?
3214         found=1
3215     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
3216
3217     ! get_bool "$found" && return 1
3218     return 0
3219 }
3220
3221 # instmods [-c] <kernel module> [<kernel module> ... ]
3222 # instmods [-c] <kernel subsystem>
3223 # install kernel modules along with all their dependencies.
3224 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
3225 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
3226 instmods() {
3227     local check=no
3228     if [[ $# -ge 0 && "$1" = '-c' ]]; then
3229         check=yes
3230         shift
3231     fi
3232
3233     inst1mod() {
3234         local mod="${1:?}"
3235         local ret=0
3236         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
3237
3238         case "$mod" in
3239             =*)
3240                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
3241                     (
3242                         [[ "$mpargs" ]] && echo "$mpargs"
3243                         cat "${mod_dir}/modules.${mod#=}"
3244                     ) | instmods
3245                 else
3246                     (
3247                         [[ "$mpargs" ]] && echo "$mpargs"
3248                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
3249                     ) | instmods
3250                 fi
3251                 ;;
3252             --*)
3253                 mpargs+=" $mod"
3254                 ;;
3255             i2o_scsi)
3256                 # Do not load this diagnostic-only module
3257                 return
3258                 ;;
3259             *)
3260                 mod=${mod##*/}
3261                 # if we are already installed, skip this module and go on
3262                 # to the next one.
3263                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
3264
3265                 # We use '-d' option in modprobe only if modules prefix path
3266                 # differs from default '/'.  This allows us to use Dracut with
3267                 # old version of modprobe which doesn't have '-d' option.
3268                 local mod_dirname=${mod_dir%%/lib/modules/*}
3269                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
3270
3271                 # ok, load the module, all its dependencies, and any firmware
3272                 # it may require
3273                 for_each_kmod_dep install_kmod_with_fw "$mod" \
3274                     --set-version "$KERNEL_VER" \
3275                     ${mod_dirname:+"$mod_dirname"} \
3276                     ${mpargs:+"$mpargs"}
3277                 ((ret+=$?))
3278                 ;;
3279         esac
3280         return "$ret"
3281     }
3282
3283     local mod mpargs
3284
3285     if [[ $# -eq 0 ]]; then  # filenames from stdin
3286         while read -r mod; do
3287             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3288                 dfatal "Failed to install $mod"
3289                 return 1
3290             fi
3291         done
3292     fi
3293
3294     for mod in "$@"; do # filenames as arguments
3295         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3296             dfatal "Failed to install $mod"
3297             return 1
3298         fi
3299     done
3300
3301     return 0
3302 }
3303
3304 _umount_dir() {
3305     local mountpoint="${1:?}"
3306     if mountpoint -q "$mountpoint"; then
3307         ddebug "umount $mountpoint"
3308         umount "$mountpoint"
3309     fi
3310 }
3311
3312 # can be overridden in specific test
3313 test_setup_cleanup() {
3314     cleanup_initdir
3315 }
3316
3317 _test_cleanup() {
3318     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3319     (
3320         set +e
3321         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3322         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3323         # If multiple setups/cleans are ran in parallel, this can cause a race
3324         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3325             rm -vf "${IMAGESTATEDIR}/default.img"
3326         fi
3327         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3328         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3329     ) || :
3330 }
3331
3332 # can be overridden in specific test
3333 test_cleanup() {
3334     _test_cleanup
3335 }
3336
3337 test_cleanup_again() {
3338     [ -n "$TESTDIR" ] || return
3339     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3340     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3341     # Test specific images are not reused, so delete them or we run out of disk space
3342     if [[ -n "$IMAGE_PUBLIC" ]] && [ "$(basename "$IMAGE_PUBLIC")" != "default.img" ]; then
3343         rm -vf "$IMAGE_PUBLIC"
3344     fi
3345     if [[ -n "$IMAGE_PRIVATE" ]] && [ "$(basename "$IMAGE_PRIVATE")" != "default.img" ]; then
3346         rm -vf "$IMAGE_PRIVATE"
3347     fi
3348 }
3349
3350 test_create_image() {
3351     create_empty_image_rootdir
3352
3353     # Create what will eventually be our root filesystem onto an overlay
3354     (
3355         LOG_LEVEL=5
3356         setup_basic_environment
3357     )
3358 }
3359
3360 test_setup() {
3361     if ! get_bool "$NO_BUILD" && \
3362          get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3363          command -v meson >/dev/null && \
3364          [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3365         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3366         exit 1
3367     fi
3368
3369     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3370         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3371         mount_initdir
3372     else
3373         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3374             # default.img is the base that every test uses and optionally appends to
3375             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3376                 # Create the backing public image, but then completely unmount
3377                 # it and drop the loopback device responsible for it, since we're
3378                 # going to symlink/copy the image and mount it again from
3379                 # elsewhere.
3380                 local image_old="${IMAGE_PUBLIC}"
3381                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3382                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3383                 fi
3384                 test_create_image
3385                 test_setup_cleanup
3386                 umount_loopback
3387                 cleanup_loopdev
3388                 IMAGE_PUBLIC="${image_old}"
3389             fi
3390             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3391                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3392             fi
3393         fi
3394
3395         local hook_defined
3396         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3397
3398         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3399         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3400             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3401         else
3402             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3403         fi
3404
3405         mount_initdir
3406
3407         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3408             dinfo "Masking supporting services"
3409             mask_supporting_services
3410         fi
3411
3412         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
3413             # Do an initial coverage capture, to make sure the final report includes
3414             # files that the tests didn't touch at all
3415             lcov --initial --capture --directory "${initdir}/${BUILD_DIR:?}" --output-file "${TESTDIR:?}/coverage-base"
3416         fi
3417
3418         if get_bool "$hook_defined"; then
3419             test_append_files "${initdir:?}"
3420         fi
3421     fi
3422
3423     setup_nspawn_root
3424 }
3425
3426 test_run() {
3427     local test_id="${1:?}"
3428     mount_initdir
3429
3430     if ! get_bool "${TEST_NO_QEMU:=}"; then
3431         if run_qemu "$test_id"; then
3432             check_result_qemu || { echo "qemu test failed"; return 1; }
3433         else
3434             dwarn "can't run qemu, skipping"
3435         fi
3436     fi
3437     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3438         mount_initdir
3439         if run_nspawn "${initdir:?}" "$test_id"; then
3440             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3441         else
3442             dwarn "can't run systemd-nspawn, skipping"
3443         fi
3444
3445         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3446             dir="$TESTDIR/unprivileged-nspawn-root"
3447             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3448                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3449             else
3450                 dwarn "can't run systemd-nspawn, skipping"
3451             fi
3452         fi
3453     fi
3454     return 0
3455 }
3456
3457 do_test() {
3458     if [[ $UID != "0" ]]; then
3459         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3460         exit 0
3461     fi
3462
3463     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3464         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3465         exit 0
3466     fi
3467
3468     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3469         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3470         exit 0
3471     fi
3472
3473     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3474         TEST_NO_QEMU=1
3475     fi
3476
3477     # Detect lib paths
3478     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3479         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3480     done
3481
3482     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3483         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3484     done
3485
3486     mkdir -p "$WORKDIR"
3487     mkdir -p "$STATEDIR"
3488
3489     import_testdir
3490     import_initdir
3491
3492     if [ -n "${SUDO_USER}" ]; then
3493         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3494         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3495     fi
3496
3497     testname="$(basename "$PWD")"
3498
3499     while (($# > 0)); do
3500         case $1 in
3501             --run)
3502                 echo "${testname} RUN: $TEST_DESCRIPTION"
3503                 test_run "$TESTID"
3504                 ret=$?
3505                 if [ $ret -eq 0 ]; then
3506                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3507                 else
3508                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3509                 fi
3510                 exit $ret
3511                 ;;
3512             --setup)
3513                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3514                 test_setup
3515                 test_setup_cleanup
3516                 ;;
3517             --clean)
3518                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3519                 test_cleanup
3520                 ;;
3521             --clean-again)
3522                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3523                 test_cleanup_again
3524                 ;;
3525             --all)
3526                 ret=0
3527                 echo -n "${testname}: $TEST_DESCRIPTION "
3528                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3529                 # and loop devices will leak
3530                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3531                 if [ $ret -eq 0 ]; then
3532                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3533                 fi
3534                 if [ $ret -eq 0 ]; then
3535                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3536                 fi
3537                 test_cleanup
3538                 if [ $ret -eq 0 ]; then
3539                     # $TESTLOG is in $STATEDIR, so clean it up only on success
3540                     [[ -n "$STATEDIR" ]] && rm -vfr "$STATEDIR"
3541                     echo "[OK]"
3542                 else
3543                     echo "[FAILED]"
3544                     echo "see $TESTLOG"
3545                 fi
3546                 exit $ret
3547                 ;;
3548             *)
3549                 break
3550                 ;;
3551         esac
3552         shift
3553     done
3554 }