test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 # Simple wrapper to unify boolean checks.
  18 # Note: this function needs to stay near the top of the file, so we can use it
  19 #       in code in the outermost scope.
  20 get_bool() {
  21     # Make the value lowercase to make the regex matching simpler
  22     local _bool="${1,,}"
  23
  24     # Consider empty value as "false"
  25     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  26         return 1
  27     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  28         return 0
  29     else
  30         echo >&2 "Value '$_bool' is not a valid boolean value"
  31         exit 1
  32     fi
  33 }
  34
  35 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  36 export PATH
  37
  38 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  39 # shellcheck source=/dev/null
  40 source "$os_release"
  41 [[ "$ID" == "debian" || " $ID_LIKE " == *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=no
  42 [[ "$ID" == "arch" || " $ID_LIKE " == *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=no
  43 [[ "$ID" == "fedora" ]] && LOOKS_LIKE_FEDORA=yes || LOOKS_LIKE_FEDORA=no
  44 [[ " $ID_LIKE " == *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=no
  45
  46 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  47 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  48 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  49 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  50 get_bool "$LOOKS_LIKE_SUSE" && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  51 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  52 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  53 # Note that defining a different IMAGE_NAME in a test setup script will only result
  54 # in default.img being copied and renamed. It can then be extended by defining
  55 # a test_append_files() function. The $1 parameter will be the root directory.
  56 # To force creating a new image from scratch (eg: to encrypt it), also define
  57 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  58 IMAGE_NAME=${IMAGE_NAME:-default}
  59 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  60 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  61 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  62 LOOPDEV=
  63
  64 # Since in Bash we can have only one handler per signal, let's overcome this
  65 # limitation by having one global handler for the EXIT signal which executes
  66 # all registered handlers
  67 _AT_EXIT_HANDLERS=()
  68 _at_exit() {
  69     set +e
  70
  71     # Run the EXIT handlers in reverse order
  72     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  73         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  74         eval "${_AT_EXIT_HANDLERS[$i]}"
  75     done
  76 }
  77
  78 trap _at_exit EXIT
  79
  80 add_at_exit_handler() {
  81     _AT_EXIT_HANDLERS+=("${1:?}")
  82 }
  83
  84 # Decide if we can (and want to) run qemu with KVM acceleration.
  85 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  86 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  87 # running under KVM. If these conditions are met, enable KVM (and possibly
  88 # nested KVM), otherwise disable it.
  89 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  90     QEMU_KVM=yes
  91 else
  92     QEMU_KVM=no
  93 fi
  94
  95 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  96     echo "WARNING! Cannot determine libdir from pkg-config, assuming /usr/lib/systemd" >&2
  97     ROOTLIBDIR=/usr/lib/systemd
  98 fi
  99
 100 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
 101 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 102 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 103 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 104 # These variables are used by test scripts
 105 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR
 106
 107 TOOLS_DIR="$SOURCE_DIR/tools"
 108 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 109 if get_bool "${NO_BUILD:=}"; then
 110     BUILD_DIR="$SOURCE_DIR"
 111 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 112     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 113     exit 1
 114 fi
 115
 116 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 117 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 118 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 119 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 120 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 121 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 122 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 123 SYSTEMD_ID128="${SYSTEMD_ID128:-$(command -v "$BUILD_DIR/systemd-id128" || command -v systemd-id128)}"
 124
 125 TESTFILE="${BASH_SOURCE[1]}"
 126 if [ -z "$TESTFILE" ]; then
 127     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 128     exit 1
 129 fi
 130 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 131
 132 WORKDIR="/var/tmp/systemd-tests"
 133 if get_bool "${NO_BUILD:=}"; then
 134     STATEDIR="$WORKDIR/$TESTNAME"
 135 else
 136     STATEDIR="$BUILD_DIR/test/$TESTNAME"
 137 fi
 138
 139 STATEFILE="$STATEDIR/.testdir"
 140 IMAGESTATEDIR="$STATEDIR/.."
 141 TESTLOG="$STATEDIR/test.log"
 142
 143 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 144     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 145     exit 1
 146 fi
 147 TESTID="${BASH_REMATCH[1]:?}"
 148
 149 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 150     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 151     exit 1
 152 fi
 153
 154 BASICTOOLS=(
 155     awk
 156     base64
 157     basename
 158     bash
 159     capsh
 160     cat
 161     chgrp
 162     chmod
 163     chown
 164     chroot
 165     cmp
 166     cp
 167     cryptsetup
 168     cut
 169     date
 170     dd
 171     dhclient
 172     diff
 173     dirname
 174     dmsetup
 175     echo
 176     env
 177     false
 178     find
 179     findmnt
 180     flock
 181     getconf
 182     getent
 183     getfacl
 184     getfattr
 185     grep
 186     grep
 187     gunzip
 188     gzip
 189     head
 190     hostname
 191     id
 192     ionice
 193     ip
 194     jq
 195     killall
 196     ldd
 197     ln
 198     ln
 199     loadkeys
 200     login
 201     losetup
 202     ls
 203     lsattr
 204     lsblk
 205     lz4cat
 206     mkdir
 207     mkfifo
 208     mknod
 209     mktemp
 210     modprobe
 211     mount
 212     mountpoint
 213     mv
 214     nc
 215     nproc
 216     ping
 217     pkill
 218     ps
 219     readlink
 220     realpath
 221     rev
 222     rm
 223     rmdir
 224     rmmod
 225     route
 226     script
 227     sed
 228     seq
 229     setfacl
 230     setfattr
 231     setfont
 232     setpriv
 233     setsid
 234     sfdisk
 235     sh
 236     sleep
 237     sort
 238     stat
 239     stty
 240     su
 241     sulogin
 242     sysctl
 243     tail
 244     tar
 245     tee
 246     test
 247     timeout
 248     touch
 249     tr
 250     true
 251     truncate
 252     tty
 253     umount
 254     uname
 255     unshare
 256     useradd
 257     userdel
 258     wc
 259     whoami
 260     xargs
 261     xzcat
 262 )
 263
 264 DEBUGTOOLS=(
 265     df
 266     dmesg
 267     du
 268     free
 269     less
 270     strace
 271     vi
 272     /usr/libexec/vi
 273 )
 274
 275 is_built_with_asan() {
 276     local _bin="${1:?}"
 277
 278     if ! type -P objdump >/dev/null; then
 279         echo "Failed to find objdump, assuming systemd hasn't been built with ASAN."
 280         return 1
 281     fi
 282
 283     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 284     local _asan_calls
 285     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 286     if ((_asan_calls < 1000)); then
 287         return 1
 288     else
 289         return 0
 290     fi
 291 }
 292
 293 is_built_with_coverage() {
 294     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 295         return 1
 296     fi
 297
 298     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 299 }
 300
 301 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 302 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 303
 304 if get_bool "$IS_BUILT_WITH_ASAN"; then
 305     PATH_TO_INIT="$ROOTLIBDIR/systemd-under-asan"
 306     QEMU_MEM="${QEMU_MEM:-2G}"
 307     QEMU_SMP="${QEMU_SMP:-4}"
 308
 309     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 310     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 311         ASAN_COMPILER=gcc
 312         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 313     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 314         ASAN_COMPILER=clang
 315         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 316
 317         # As clang's ASan DSO is usually in a non-standard path, let's check if
 318         # the environment is set accordingly. If not, warn the user and exit.
 319         # We're not setting the LD_LIBRARY_PATH automagically here, because
 320         # user should encounter (and fix) the same issue when running the unit
 321         # tests (meson test)
 322         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 323             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 324             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 325             exit 1
 326         fi
 327     else
 328         echo >&2 "systemd is not linked against the ASan DSO"
 329         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 330         exit 1
 331     fi
 332
 333     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 334 fi
 335
 336 test_require_bin() {
 337     local bin
 338
 339     for bin in "$@"; do
 340         if ! command -v "$bin" >/dev/null; then
 341             echo "Required binary $bin not available, skipping the test"
 342             exit 0
 343         fi
 344     done
 345 }
 346
 347 find_qemu_bin() {
 348     QEMU_BIN="${QEMU_BIN:-""}"
 349     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 350     if get_bool "$QEMU_KVM"; then
 351         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 352     fi
 353
 354     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 355     case $ARCH in
 356     x86_64)
 357         # QEMU's own build system calls it qemu-system-x86_64
 358         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 359         ;;
 360     i*86)
 361         # new i386 version of QEMU
 362         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 363
 364         # i386 version of QEMU
 365         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 366         ;;
 367     ppc64*)
 368         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 369         ;;
 370     esac
 371
 372     if [[ ! -e "$QEMU_BIN" ]]; then
 373         echo "Could not find a suitable qemu binary" >&2
 374         return 1
 375     fi
 376 }
 377
 378 qemu_setup_swtpm_socket() {
 379     local pid state_dir tpm_device
 380
 381     if ! tpm_device="$(qemu_get_tpm_device)"; then
 382         dinfo "Found QEMU version is too old for TPM2 on ppc64le"
 383         exit 0
 384     fi
 385
 386     state_dir="$(mktemp -d)"
 387     swtpm socket --tpm2 --tpmstate dir="$state_dir" --ctrl type=unixio,path="$state_dir/sock" &
 388     pid=$!
 389     if ! kill -0 "$pid"; then
 390         derror "Failed to start swtpm"
 391         return 1
 392     fi
 393
 394     if ! timeout 5 bash -c "until [[ -S $state_dir/sock ]]; do sleep .5; done"; then
 395         derror "Failed to setup swtpm socket"
 396         return 1
 397     fi
 398
 399     dinfo "Started swtpm as PID $pid with state dir $state_dir"
 400
 401     add_at_exit_handler "kill -TERM $pid 2>/dev/null; rm -rf '$state_dir'"
 402
 403     QEMU_OPTIONS+=" -chardev socket,id=chrtpm,path=$state_dir/sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device $tpm_device,tpmdev=tpm0"
 404     dinfo "Configured emulated TPM2 device $tpm_device"
 405
 406     return 0
 407 }
 408
 409 qemu_get_tpm_device() {
 410     local tpm_device="tpm-tis"
 411
 412     if [[ "$(uname -m)" == "ppc64le" ]]; then
 413         # tpm-spapr support was introduced in qemu 5.0.0
 414         if ! qemu_min_version "5.0.0"; then
 415             return 1
 416         fi
 417
 418         tpm_device="tpm-spapr"
 419     fi
 420
 421     echo "$tpm_device"
 422     return 0
 423 }
 424
 425 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 426 # returns 0 if newer or equal
 427 # returns 1 if older
 428 # returns 2 if failing
 429 qemu_min_version() {
 430     find_qemu_bin || return 2
 431
 432     # get version from binary
 433     local qemu_ver
 434     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 435
 436     # Check version string format
 437     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 438     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 439
 440     # compare as last command to return that value
 441     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 442 }
 443
 444 # Pads a file to multiple of 4 bytes
 445 pad4_file() {
 446     local size
 447     size=$(stat -c "%s" "$1")
 448     local padded
 449     padded=$((((size + 3) / 4) * 4))
 450     truncate -s "$padded" "$1"
 451 }
 452
 453 # Return 0 if qemu did run (then you must check the result state/logs for actual
 454 # success), or 1 if qemu is not available.
 455 run_qemu() {
 456     if declare -F run_qemu_hook >/dev/null; then
 457         if ! run_qemu_hook "${workspace}"; then
 458             derror "check_qemu_hook() returned with EC > 0"
 459             ret=4
 460         fi
 461     fi
 462
 463     # If the test provided its own initrd, use it (e.g. TEST-24)
 464     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 465         INITRD="$TESTDIR/initrd.img"
 466     fi
 467
 468     if [ -f /etc/machine-id ]; then
 469         read -r MACHINE_ID </etc/machine-id
 470         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 471             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 472         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 473             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 474     fi
 475
 476     local CONSOLE=ttyS0
 477
 478     # Reset the boot counter, if present
 479     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 480     rm -f "$initdir"/{testok,failed,skipped}
 481     # make sure the initdir is not mounted to avoid concurrent access
 482     cleanup_initdir
 483     umount_loopback
 484
 485     if [[ ! "$KERNEL_BIN" ]]; then
 486         if get_bool "$LOOKS_LIKE_ARCH"; then
 487             KERNEL_BIN=/boot/vmlinuz-linux
 488         else
 489             [ "$ARCH" ] || ARCH=$(uname -m)
 490             case $ARCH in
 491                 ppc64*)
 492                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 493                     # uses the "standard" vmlinuz- prefix
 494                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 495                     CONSOLE=hvc0
 496                     ;;
 497                 *)
 498                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 499                     ;;
 500             esac
 501         fi
 502     fi
 503
 504     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 505     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 506     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 507     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 508     if [[ ! "$INITRD" ]]; then
 509         if [[ -e "$default_fedora_initrd" ]]; then
 510             INITRD="$default_fedora_initrd"
 511         elif get_bool "$LOOKS_LIKE_DEBIAN" && [[ -e "$default_debian_initrd" ]]; then
 512             INITRD="$default_debian_initrd"
 513         elif get_bool "$LOOKS_LIKE_ARCH" && [[ -e "$default_arch_initrd" ]]; then
 514             INITRD="$default_arch_initrd"
 515         elif get_bool "$LOOKS_LIKE_SUSE" && [[ -e "$default_suse_initrd" ]]; then
 516             INITRD="$default_suse_initrd"
 517         fi
 518     fi
 519
 520     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 521     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 522     # is not installed or there's some other error when calling it, fall back
 523     # to the original value (QEMU_SMP=1).
 524     if [[ -z "${QEMU_SMP:=}" ]]; then
 525         if ! QEMU_SMP=$(nproc); then
 526             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 527             QEMU_SMP=1
 528         fi
 529     fi
 530
 531     find_qemu_bin || return 1
 532
 533     if get_bool "${TEST_SETUP_SWTPM:-}"; then
 534         qemu_setup_swtpm_socket || return 1
 535     fi
 536
 537     # Umount initdir to avoid concurrent access to the filesystem
 538     _umount_dir "$initdir"
 539
 540     local kernel_params=()
 541     local qemu_options=()
 542     local qemu_cmd=("$QEMU_BIN")
 543
 544     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 545         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 546     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 547         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 548     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 549         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 550     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 551         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 552         exit 1
 553     fi
 554
 555     if get_bool "$LOOKS_LIKE_SUSE"; then
 556         kernel_params+=("rd.hostonly=0")
 557     fi
 558
 559     # Debian/Ubuntu's initramfs tries to check if it can resume from hibernation
 560     # and wastes a minute or so probing disks, skip that as it's not useful here
 561     kernel_params+=(
 562         "root=LABEL=systemd_boot"
 563         "rw"
 564         "raid=noautodetect"
 565         "rd.luks=0"
 566         "loglevel=2"
 567         "init=$PATH_TO_INIT"
 568         "console=$CONSOLE"
 569         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 570         "systemd.unit=testsuite.target"
 571         "systemd.wants=testsuite-$1.service"
 572         "noresume"
 573         "oops=panic"
 574         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 575         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 576     )
 577
 578     if ! get_bool "$INTERACTIVE_DEBUG" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 579         kernel_params+=(
 580             "panic=1"
 581             "softlockup_panic=1"
 582             "systemd.wants=end.service"
 583         )
 584     fi
 585
 586     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 587     qemu_options+=(
 588         -smp "$QEMU_SMP"
 589         -net none
 590         -m "${QEMU_MEM:-768M}"
 591         -nographic
 592         -kernel "$KERNEL_BIN"
 593         -drive "format=raw,cache=unsafe,file=$image"
 594         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 595     )
 596
 597     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 598         local user_qemu_options
 599         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 600         qemu_options+=("${user_qemu_options[@]}")
 601     fi
 602     qemu_options+=(${QEMU_OPTIONS_ARRAY:+"${QEMU_OPTIONS_ARRAY[@]}"})
 603
 604     if [[ -n "$INITRD" ]]; then
 605         if [[ -n "$INITRD_EXTRA" ]]; then
 606             # An addition initrd has been specified, let's combine it with the main one.
 607             local t="$WORKDIR"/initrd.combined."$RANDOM"
 608
 609             # First, show contents of additional initrd
 610             echo "Additional initrd contents:"
 611             cpio -tv < "$INITRD_EXTRA"
 612
 613             # Copy the main initrd
 614             zstd -d -c -f "$INITRD" > "$t"
 615             add_at_exit_handler "rm $t"
 616             # Kernel requires this to be padded to multiple of 4 bytes with zeroes
 617             pad4_file "$t"
 618
 619             # Copy the additional initrd
 620             cat "$INITRD_EXTRA" >> "$t"
 621             pad4_file "$t"
 622
 623             qemu_options+=(-initrd "$t")
 624         else
 625             qemu_options+=(-initrd "$INITRD")
 626         fi
 627     fi
 628
 629     # Let's use KVM if possible
 630     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 631         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 632     fi
 633
 634     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 635         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 636     fi
 637
 638     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]} ${KERNEL_APPEND:-}" |& tee "${TESTDIR:?}/console.log")
 639     rc=$?
 640     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 641         derror "Test timed out after ${QEMU_TIMEOUT}s"
 642         TIMED_OUT=1
 643     else
 644         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 645     fi
 646     return 0
 647 }
 648
 649 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 650 # success), or 1 if nspawn is not available.
 651 run_nspawn() {
 652     [[ -d /run/systemd/system ]] || return 1
 653     # Reset the boot counter, if present
 654     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 655     rm -f "${initdir:?}"/{testok,failed,skipped}
 656
 657     local nspawn_cmd=()
 658     local nspawn_options=(
 659         "--register=no"
 660         "--kill-signal=SIGKILL"
 661         "--directory=${1:?}"
 662         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 663         "--machine=TEST-$TESTID"
 664     )
 665     local kernel_params=(
 666         "$PATH_TO_INIT"
 667         "systemd.unit=testsuite.target"
 668         "systemd.wants=testsuite-$2.service"
 669         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 670         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 671     )
 672
 673     if get_bool "$INTERACTIVE_DEBUG"; then
 674         nspawn_options+=("--console=interactive")
 675     elif ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 676         kernel_params+=("systemd.wants=end.service")
 677     fi
 678
 679     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 680         local user_nspawn_arguments
 681         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 682         nspawn_options+=("${user_nspawn_arguments[@]}")
 683     fi
 684
 685     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 686         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 687         exit
 688     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 689         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 690     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 691         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 692     else
 693         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 694         exit 1
 695     fi
 696
 697     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 698         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 699     else
 700         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 701     fi
 702
 703     # Word splitting here is intentional
 704     # shellcheck disable=SC2086
 705     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}" ${KERNEL_APPEND:-} |& tee "${TESTDIR:?}/console.log")
 706     rc=$?
 707     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 708         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 709         TIMED_OUT=1
 710     else
 711         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 712     fi
 713     return 0
 714 }
 715
 716 # Build two very minimal root images, with two units, one is the same and one is different across them
 717 install_verity_minimal() {
 718     dinfo "Set up a set of minimal images for verity verification"
 719     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 720         return
 721     fi
 722     if ! command -v mksquashfs >/dev/null 2>&1; then
 723         dfatal "mksquashfs not found"
 724         exit 1
 725     fi
 726     if ! command -v veritysetup >/dev/null 2>&1; then
 727         dfatal "veritysetup not found"
 728         exit 1
 729     fi
 730     # Local modifications of some global variables is intentional in this
 731     # subshell (SC2030)
 732     # shellcheck disable=SC2030
 733     (
 734         BASICTOOLS=(
 735             bash
 736             cat
 737             grep
 738             mount
 739             sleep
 740             touch
 741         )
 742         oldinitdir="$initdir"
 743         rm -rfv "$TESTDIR/minimal"
 744         export initdir="$TESTDIR/minimal"
 745         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 746         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 747         setup_basic_dirs
 748         install_basic_tools
 749         install_ld_so_conf
 750         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 751         # issue, thus falsely triggering SC2030 in this case
 752         # See: koalaman/shellcheck#1409
 753         if [[ -v ASAN_RT_PATH ]]; then
 754             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 755             # into the verity images to get rid of the annoying errors about
 756             # missing $LD_PRELOAD libraries.
 757             inst_libs "$ASAN_RT_PATH"
 758             inst_library "$ASAN_RT_PATH"
 759         fi
 760         cp "$os_release" "$initdir/usr/lib/os-release"
 761         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 762         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 763         touch "$initdir/opt/some_file"
 764         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 765         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 766         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 767 [Service]
 768 ExecStartPre=cat /usr/lib/os-release
 769 ExecStart=sleep 120
 770 EOF
 771         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 772
 773         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 774         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 775             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 776
 777         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 778         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 779         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 780
 781         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 782         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 783             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 784
 785         # Rolling distros like Arch do not set VERSION_ID
 786         local version_id=""
 787         if grep -q "^VERSION_ID=" "$os_release"; then
 788             version_id="$(grep "^VERSION_ID=" "$os_release")"
 789         fi
 790
 791         export initdir="$TESTDIR/app0"
 792         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 793         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 794         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 795         ( echo "${version_id}"
 796           echo "SYSEXT_IMAGE_ID=app" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 797         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 798 [Service]
 799 Type=oneshot
 800 RemainAfterExit=yes
 801 ExecStart=/opt/script0.sh
 802 TemporaryFileSystem=/var/lib
 803 StateDirectory=app0
 804 RuntimeDirectory=app0
 805 EOF
 806         cat >"$initdir/opt/script0.sh" <<EOF
 807 #!/bin/bash
 808 set -e
 809 test -e /usr/lib/os-release
 810 echo bar >\${STATE_DIRECTORY}/foo
 811 cat /usr/lib/extension-release.d/extension-release.app0
 812 EOF
 813         chmod +x "$initdir/opt/script0.sh"
 814         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 815         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 816
 817         export initdir="$TESTDIR/conf0"
 818         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system" "$initdir/opt"
 819         grep "^ID=" "$os_release" >"$initdir/etc/extension-release.d/extension-release.conf0"
 820         echo "${version_id}" >>"$initdir/etc/extension-release.d/extension-release.conf0"
 821         ( echo "${version_id}"
 822           echo "CONFEXT_IMAGE_ID=app" ) >>"$initdir/etc/extension-release.d/extension-release.conf0"
 823         echo MARKER_1 >"$initdir/etc/systemd/system/some_file"
 824         mksquashfs "$initdir" "$oldinitdir/usr/share/conf0.raw" -noappend
 825
 826         export initdir="$TESTDIR/app1"
 827         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 828         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 829         ( echo "${version_id}"
 830           echo "SYSEXT_SCOPE=portable"
 831           echo "SYSEXT_IMAGE_ID=app"
 832           echo "SYSEXT_IMAGE_VERSION=1"
 833           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 834         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 835         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 836 [Service]
 837 Type=oneshot
 838 RemainAfterExit=yes
 839 ExecStart=/opt/script1.sh
 840 StateDirectory=app1
 841 RuntimeDirectory=app1
 842 EOF
 843         cat >"$initdir/opt/script1.sh" <<EOF
 844 #!/bin/bash
 845 set -e
 846 test -e /usr/lib/os-release
 847 echo baz >\${STATE_DIRECTORY}/foo
 848 cat /usr/lib/extension-release.d/extension-release.app2
 849 EOF
 850         chmod +x "$initdir/opt/script1.sh"
 851         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 852         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 853
 854         export initdir="$TESTDIR/app-nodistro"
 855         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 856         ( echo "ID=_any"
 857           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 858         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 859         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 860
 861         export initdir="$TESTDIR/service-scoped-test"
 862         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system"
 863         ( echo "ID=_any"
 864           echo "ARCHITECTURE=_any" ) >"$initdir/etc/extension-release.d/extension-release.service-scoped-test"
 865         echo MARKER_CONFEXT_123 >"$initdir/etc/systemd/system/some_file"
 866         mksquashfs "$initdir" "$oldinitdir/etc/service-scoped-test.raw" -noappend
 867
 868         # We need to create a dedicated sysext image to test the reload mechanism. If we share an image to install the
 869         # 'foo.service' it will be loaded from another test run, which will impact the targeted test.
 870         export initdir="$TESTDIR/app-reload"
 871         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 872         ( echo "ID=_any"
 873           echo "ARCHITECTURE=_any"
 874           echo "EXTENSION_RELOAD_MANAGER=1" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-reload"
 875         mkdir -p "$initdir/usr/lib/systemd/system/multi-user.target.d"
 876         cat >"${initdir}/usr/lib/systemd/system/foo.service" <<EOF
 877 [Service]
 878 Type=oneshot
 879 RemainAfterExit=yes
 880 ExecStart=echo foo
 881
 882 [Install]
 883 WantedBy=multi-user.target
 884 EOF
 885         { echo "[Unit]"; echo "Upholds=foo.service"; } > "$initdir/usr/lib/systemd/system/multi-user.target.d/10-foo-service.conf"
 886         mksquashfs "$initdir" "$oldinitdir/usr/share/app-reload.raw" -noappend
 887     )
 888 }
 889
 890 setup_basic_environment() {
 891     # create the basic filesystem layout
 892     setup_basic_dirs
 893
 894     install_systemd
 895     install_missing_libraries
 896     install_config_files
 897     install_zoneinfo
 898     create_rc_local
 899     install_basic_tools
 900     install_libnss
 901     install_pam
 902     install_dbus
 903     install_fonts
 904     install_locales
 905     install_keymaps
 906     install_x11_keymaps
 907     install_terminfo
 908     install_execs
 909     install_fs_tools
 910     install_modules
 911     install_plymouth
 912     install_haveged
 913     install_debug_tools
 914     install_ld_so_conf
 915     install_testuser
 916     has_user_dbus_socket && install_user_dbus
 917     setup_selinux
 918     install_depmod_files
 919     generate_module_dependencies
 920     if get_bool "$IS_BUILT_WITH_ASAN"; then
 921         create_asan_wrapper
 922     fi
 923     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 924         install_verity_minimal
 925     fi
 926 }
 927
 928 setup_selinux() {
 929     dinfo "Setup SELinux"
 930     # don't forget KERNEL_APPEND='... selinux=1 ...'
 931     if ! get_bool "$SETUP_SELINUX"; then
 932         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 933         return 0
 934     fi
 935
 936     for dir in /etc/selinux /usr/share/selinux; do
 937         rm -rf "${initdir:?}/$dir"
 938         if ! cp -ar "$dir" "$initdir/$dir"; then
 939             dfatal "Failed to copy $dir"
 940             exit 1
 941         fi
 942     done
 943
 944     # We use a custom autorelabel service instead of the SELinux provided set
 945     # of units & a generator, since the generator overrides the default target
 946     # to the SELinux one when it detects /.autorelabel. However, we use
 947     # systemd.unit= on the kernel command cmdline which always takes precedence,
 948     # rendering all SELinux efforts useless. Also, pulling in selinux-autorelabel.service
 949     # explicitly doesn't work either, as it doesn't check for the presence of /.autorelabel
 950     # and does the relabeling unconditionally which always ends with a reboot, so
 951     # we end up in a reboot loop (and it also spews quite a lot of errors as it
 952     # wants /etc/fstab and dracut-initramfs-restore).
 953     touch "$initdir/.autorelabel"
 954     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 955     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 956
 957     # Tools requires by fixfiles
 958     image_install awk bash cat chcon expr egrep find grep head secon setfiles rm sort uname uniq
 959     image_install fixfiles getenforce load_policy selinuxenabled sestatus
 960 }
 961
 962 install_valgrind() {
 963     if ! type -p valgrind; then
 964         dfatal "Failed to install valgrind"
 965         exit 1
 966     fi
 967
 968     local valgrind_bins valgrind_libs valgrind_supp
 969
 970     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null |
 971                                        sed -r -n 's/execve\("([^"]*)".*/\1/p')
 972     image_install "${valgrind_bins[@]}"
 973
 974     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null |
 975                                        sed -r -n 's|.*calling init: (/.*vgpreload_.*)|\1|p')
 976     image_install "${valgrind_libs[@]}"
 977
 978     readarray -t valgrind_supp < <(strace -e open valgrind /bin/true 2>&1 >/dev/null |
 979                                        sed -r -n 's,open\("([^"]*(/debug[^"]*|\.supp))".*= [0-9].*,\1,p')
 980     image_install "${valgrind_supp[@]}"
 981 }
 982
 983 create_valgrind_wrapper() {
 984     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 985     ddebug "Create $valgrind_wrapper"
 986     cat >"$valgrind_wrapper" <<EOF
 987 #!/usr/bin/env bash
 988
 989 mount -t proc proc /proc
 990 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 991 EOF
 992     chmod 0755 "$valgrind_wrapper"
 993 }
 994
 995 create_asan_wrapper() {
 996     local asan_wrapper default_asan_options default_ubsan_options default_environment manager_environment
 997
 998     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 999
1000     default_asan_options="${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1}"
1001     default_ubsan_options="${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}"
1002
1003     if [[ "$ASAN_COMPILER" == "clang" ]]; then
1004         # clang: install llvm-symbolizer to generate useful reports
1005         # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
1006         image_install "llvm-symbolizer"
1007
1008         # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
1009         # unnecessary for gcc & libasan, however, for clang this is crucial, as its
1010         # runtime ASan DSO is in a non-standard (library) path.
1011         mkdir -p "${initdir:?}/etc/ld.so.conf.d/"
1012         echo "${ASAN_RT_PATH%/*}" >"${initdir:?}/etc/ld.so.conf.d/asan-path-override.conf"
1013         ldconfig -r "$initdir"
1014     fi
1015
1016     # Create a simple environment file which can be included by systemd services
1017     # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
1018     cat >"${initdir:?}/usr/lib/systemd/systemd-asan-env" <<EOF
1019 LD_PRELOAD=$ASAN_RT_PATH
1020 ASAN_OPTIONS=$default_asan_options
1021 LSAN_OPTIONS=detect_leaks=0
1022 UBSAN_OPTIONS=$default_ubsan_options
1023 EOF
1024
1025     default_environment=(
1026         "ASAN_OPTIONS='$default_asan_options'"
1027         "UBSAN_OPTIONS='$default_ubsan_options'"
1028         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1029     )
1030     manager_environment=(
1031         "ASAN_OPTIONS='$default_asan_options:log_path=/systemd-pid1.asan.log:log_to_syslog=1'"
1032         "UBSAN_OPTIONS='$default_ubsan_options:log_path=/systemd-pid1.ubsan.log:log_to_syslog=1'"
1033         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1034     )
1035
1036     mkdir -p "${initdir:?}/etc/systemd/system.conf.d/"
1037     cat >"${initdir:?}/etc/systemd/system.conf.d/asan.conf" <<EOF
1038 [Manager]
1039 DefaultEnvironment=${default_environment[*]}
1040 ManagerEnvironment=${manager_environment[*]}
1041 DefaultTimeoutStartSec=180s
1042 EOF
1043
1044     # ASAN and syscall filters aren't compatible with each other.
1045     find "${initdir:?}" -name '*.service' -type f -print0 | xargs -0 sed -i 's/^\(MemoryDeny\|SystemCall\)/#\1/'
1046
1047     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journald.service.d/"
1048     cat >"${initdir:?}/etc/systemd/system/systemd-journald.service.d/asan-env.conf" <<EOF
1049 [Service]
1050 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
1051 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
1052 Environment=ASAN_OPTIONS=$default_asan_options:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=$default_ubsan_options:log_path=/systemd-journald.ubsan.log
1053
1054 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
1055 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
1056 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
1057 StandardOutput=file:/systemd-journald.out
1058 EOF
1059
1060     # 90s isn't enough for some services to finish when literally everything is run
1061     # under ASan+UBSan in containers, which, in turn, are run in VMs.
1062     # Let's limit which environments such services should be executed in.
1063     mkdir -p "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/"
1064     cat >"${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/asan.conf" <<EOF
1065 [Unit]
1066 ConditionVirtualization=container
1067
1068 [Service]
1069 TimeoutSec=240s
1070 EOF
1071
1072     # Let's override another hard-coded timeout that kicks in too early
1073     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/"
1074     cat >"${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/asan.conf" <<EOF
1075 [Service]
1076 TimeoutSec=180s
1077 EOF
1078
1079     asan_wrapper="${initdir:?}/${PATH_TO_INIT:?}"
1080     # Sanity check to make sure we don't overwrite something we shouldn't.
1081     [[ "$asan_wrapper" =~ systemd-under-asan$ ]]
1082
1083     cat >"$asan_wrapper" <<EOF
1084 #!/usr/bin/env bash
1085 set -eux
1086
1087 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
1088 export ${manager_environment[@]}
1089 [[ -n "\$ASAN_OPTIONS" && -n "\$UBSAN_OPTIONS" ]]
1090
1091 exec "$ROOTLIBDIR/systemd" "\$@"
1092 EOF
1093     chmod 0755 "$asan_wrapper"
1094 }
1095
1096 create_strace_wrapper() {
1097     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
1098     ddebug "Create $strace_wrapper"
1099     cat >"$strace_wrapper" <<EOF
1100 #!/usr/bin/env bash
1101
1102 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
1103 EOF
1104     chmod 0755 "$strace_wrapper"
1105 }
1106
1107 install_fs_tools() {
1108     dinfo "Install fsck"
1109     image_install /sbin/fsck*
1110     image_install -o /bin/fsck*
1111
1112     # fskc.reiserfs calls reiserfsck. so, install it
1113     image_install -o reiserfsck
1114
1115     # we use mkfs in system-repart tests
1116     image_install /sbin/mkfs.ext4
1117     image_install /sbin/mkfs.vfat
1118 }
1119
1120 install_modules() {
1121     dinfo "Install modules"
1122
1123     instmods bridge dummy ext4 ipvlan macvlan vfat veth
1124     instmods loop =block
1125     instmods nls_ascii =nls
1126     instmods overlay =overlayfs
1127     instmods scsi_debug
1128
1129     if get_bool "$LOOKS_LIKE_SUSE"; then
1130         instmods af_packet
1131     fi
1132 }
1133
1134 install_dmevent() {
1135     instmods dm_crypt =crypto
1136     inst_binary dmeventd
1137     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
1138     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1139         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
1140         # and since buster/bionic 95-dm-notify.rules
1141         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
1142         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
1143     else
1144         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
1145     fi
1146     if get_bool "$LOOKS_LIKE_SUSE"; then
1147         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
1148     fi
1149 }
1150
1151 install_multipath() {
1152     instmods "=md" multipath
1153     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
1154     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
1155     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1156         # Note: try both 60-kpartx.rules (as seen on Debian Sid with 0.9.4-7) and 95-kpartx.rules (as seen on
1157         # Ubuntu Jammy with 0.8.8-1ubuntu1.22.04.4)
1158         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-kpartx.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
1159     else
1160         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
1161     fi
1162     mkdir -p "${initdir:?}/etc/multipath"
1163
1164     local file
1165     while read -r file; do
1166         # Install libraries required by the given library
1167         inst_libs "$file"
1168         # Install the library itself and create necessary symlinks
1169         inst_library "$file"
1170     done < <(find /lib*/multipath -type f)
1171 }
1172
1173 install_lvm() {
1174     local lvm_rules rule_prefix
1175
1176     image_install lvm
1177     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1178     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1179     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1180
1181     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1182         lvm_rules="56-lvm.rules"
1183         rule_prefix=""
1184     else
1185         lvm_rules="11-dm-lvm.rules"
1186         rule_prefix="dm-"
1187     fi
1188
1189     # Support the new udev autoactivation introduced in lvm 2.03.14
1190     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1191     # Static autoactivation (via lvm2-activation-generator) was dropped
1192     # in lvm 2.03.15
1193     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1194     if [[ -f "/lib/udev/rules.d/69-${rule_prefix}lvm.rules" ]]; then
1195         inst_rules "$lvm_rules" "69-${rule_prefix}lvm.rules"
1196     else
1197         image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1198         image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1199         inst_rules "$lvm_rules" "69-${rule_prefix}lvm-metad.rules"
1200     fi
1201
1202     mkdir -p "${initdir:?}/etc/lvm"
1203 }
1204
1205 host_has_btrfs() (
1206     set -e
1207     modprobe -nv btrfs && command -v mkfs.btrfs && command -v btrfs || return $?
1208 )
1209
1210 install_btrfs() {
1211     instmods btrfs
1212     # Not all utilities provided by btrfs-progs are listed here; extend the list
1213     # if necessary
1214     image_install btrfs btrfstune mkfs.btrfs
1215     inst_rules 64-btrfs-dm.rules
1216 }
1217
1218 install_iscsi() {
1219     # Install both client and server side stuff by default
1220     local inst="${1:-}"
1221     local file
1222
1223     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1224     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1225     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1226         image_install iscsi-iname iscsiadm iscsid iscsistart
1227         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1228         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1229         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1230         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1231         mkdir -p "${initdir:?}/etc/iscsi"
1232         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1233         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1234         # a one-time service instead of distro package's post-install scripts.
1235         # However, some distros still use this approach even after this patch,
1236         # so prefer the already existing initiatorname.iscsi file if it exists.
1237         #
1238         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1239         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1240             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1241             if get_bool "$IS_BUILT_WITH_ASAN"; then
1242                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1243                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1244                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1245                 # (among others) pre-loads ASan's DSO.
1246                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1247                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1248             fi
1249         else
1250             inst_simple "/etc/iscsi/initiatorname.iscsi"
1251         fi
1252     fi
1253
1254     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1255     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1256     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1257         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1258         image_install -o /etc/sysconfig/tgtd
1259         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1260         mkdir -p "${initdir:?}/etc/tgt"
1261         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1262         # Install perl modules required by tgt-admin
1263         #
1264         # Forgive me father for I have sinned. The monstrosity below appends
1265         # a perl snippet to the `tgt-admin` perl script on the fly, which
1266         # dumps a list of files (perl modules) required by `tgt-admin` at
1267         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1268         # passed to `inst_simple` which installs the necessary files into the image
1269         #
1270         # shellcheck disable=SC2016
1271         while read -r file; do
1272             inst_simple "$file"
1273         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1274     fi
1275 }
1276
1277 host_has_mdadm() (
1278     set -e
1279     command -v mdadm || return $?
1280 )
1281
1282 install_mdadm() {
1283     local unit
1284     local mdadm_units=(
1285         system/mdadm-grow-continue@.service
1286         system/mdadm-last-resort@.service
1287         system/mdadm-last-resort@.timer
1288         system/mdmon@.service
1289         system/mdmonitor-oneshot.service
1290         system/mdmonitor-oneshot.timer
1291         system/mdmonitor.service
1292         system-shutdown/mdadm.shutdown
1293     )
1294
1295     instmods "=md"
1296     image_install mdadm mdmon
1297     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1298     # Fedora/CentOS/RHEL ships this rule file
1299     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1300
1301     for unit in "${mdadm_units[@]}"; do
1302         image_install "${ROOTLIBDIR:?}/$unit"
1303     done
1304
1305     # Disable the mdmonitor service, since it fails if there's no valid email address
1306     # configured in /etc/mdadm.conf, which just unnecessarily pollutes the logs
1307     "${SYSTEMCTL:?}" mask --root "${initdir:?}" mdmonitor.service || :
1308 }
1309
1310 install_compiled_systemd() {
1311     dinfo "Install compiled systemd"
1312
1313     local ninja_bin
1314     ninja_bin="$(type -P ninja || type -P ninja-build)"
1315     if [[ -z "$ninja_bin" ]]; then
1316         dfatal "ninja was not found"
1317         exit 1
1318     fi
1319     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1320
1321     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1322     # find them in the same directory as the runtime data counts
1323     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1324         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1325         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1326         # Set effective & default ACLs for the build dir so unprivileged
1327         # processes can write gcda files with coverage stats
1328         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1329     fi
1330 }
1331
1332 install_package_file() {
1333     local file="${1:?}"
1334
1335     # Skip missing files (like /etc/machine-info)
1336     [[ ! -e "$file" ]] && return 0
1337     # Skip python unit tests, since the image_install machinery will try to pull
1338     # in the whole python stack in a very questionable state, making the tests fail.
1339     # And given we're trying to transition to mkosi-based images anyway I'm not even
1340     # going to bother
1341     [[ "$file" =~ /tests/unit-tests/.*.py$ ]] && return 0
1342     # If the current file is a directory, create it with the original
1343     # mode; if it's a symlink to a directory, copy it as-is
1344     if [[ -d "$file" ]]; then
1345         inst_dir "$file"
1346     else
1347         inst "$file"
1348     fi
1349 }
1350
1351 install_debian_systemd() {
1352     dinfo "Install debian systemd"
1353
1354     local deb file
1355
1356     while read -r deb; do
1357         ddebug "Install debian files from package $deb"
1358         while read -r file; do
1359             install_package_file "$file"
1360         done < <(dpkg-query -L "$deb" 2>/dev/null)
1361     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1362 }
1363
1364 install_rpm() {
1365     local rpm="${1:?}"
1366     local file
1367
1368     if ! rpm -q "$rpm" >/dev/null; then
1369         derror "RPM $rpm is not installed"
1370         return 1
1371     fi
1372
1373     dinfo "Installing contents of RPM $rpm"
1374     while read -r file; do
1375         install_package_file "$file"
1376     done < <(rpm -ql "$rpm")
1377 }
1378
1379 install_suse_systemd() {
1380     local pkgs
1381
1382     dinfo "Install basic filesystem structure"
1383     install_rpm filesystem
1384
1385     dinfo "Install SUSE systemd"
1386
1387     pkgs=(
1388         systemd
1389         systemd-boot
1390         systemd-container
1391         systemd-coredump
1392         systemd-experimental
1393         systemd-homed
1394         systemd-journal-remote
1395         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1396         # systemd-network sub-package on openSUSE, has its own testsuite.
1397         systemd-network
1398         systemd-portable
1399         udev
1400     )
1401
1402     for p in "${pkgs[@]}"; do
1403         rpm -q "$p" &>/dev/null || continue
1404
1405         install_rpm "$p"
1406     done
1407
1408     dinfo "Install the data needed by the tests at runtime"
1409     inst_recursive "${SOURCE_DIR}/testdata"
1410     inst_recursive "${SOURCE_DIR}/unit-tests/manual"
1411
1412     # On openSUSE, this directory is not created at package install, at least
1413     # for now.
1414     mkdir -p "$initdir/var/log/journal/remote"
1415 }
1416
1417 install_fedora_systemd() {
1418     local required_packages=(
1419         systemd
1420         systemd-container
1421         systemd-libs
1422         systemd-pam
1423         systemd-tests
1424         systemd-udev
1425     )
1426     local optional_packages=(
1427         systemd-boot-unsigned
1428         systemd-bootchart
1429         systemd-journal-remote
1430         systemd-networkd
1431         systemd-oomd-defaults
1432         systemd-resolved
1433     )
1434     local package
1435
1436     for package in "${required_packages[@]}"; do
1437         install_rpm "$package"
1438     done
1439
1440     for package in "${optional_packages[@]}"; do
1441         rpm -q "$package" >/dev/null || continue
1442         install_rpm "$package"
1443     done
1444 }
1445
1446 install_distro_systemd() {
1447     dinfo "Install distro systemd"
1448
1449     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1450         install_debian_systemd
1451     elif get_bool "$LOOKS_LIKE_SUSE"; then
1452         install_suse_systemd
1453     elif get_bool "$LOOKS_LIKE_FEDORA"; then
1454         install_fedora_systemd
1455     else
1456         dfatal "NO_BUILD not supported for this distro"
1457         exit 1
1458     fi
1459 }
1460
1461 install_systemd() {
1462     dinfo "Install systemd"
1463     if get_bool "$NO_BUILD"; then
1464         install_distro_systemd
1465     else
1466         install_compiled_systemd
1467     fi
1468
1469     # Remove unneeded documentation
1470     rm -fr "${initdir:?}"/usr/share/{man,doc}
1471
1472     # Enable debug logging in PID1
1473     mkdir -p "$initdir/etc/systemd/system.conf.d/"
1474     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/system.conf.d/10-log-level.conf"
1475     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1476         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf.d/99-log-level.conf"
1477     fi
1478     # Enable debug logging for user instances as well
1479     mkdir -p "$initdir/etc/systemd/user.conf.d/"
1480     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/user.conf.d/10-log-level.conf"
1481     # Store coredumps in journal
1482     mkdir -p "$initdir/etc/systemd/coredump.conf.d/"
1483     echo -ne "[Coredump]\nStorage=journal\n" >"$initdir/etc/systemd/coredump.conf.d/10-storage-journal.conf"
1484     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1485     mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
1486     echo -ne "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/99-SYSTEMD_UNIT_PATH.conf"
1487
1488     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1489     # images, since it prevents gcov to write the coverage reports (*.gcda
1490     # files)
1491     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1492         mkdir -p "$initdir/etc/systemd/system/service.d/"
1493         echo -ne "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1494         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image to make the coverage work with
1495         # units using DynamicUser=yes. Do this only for services with test- prefix and a couple of
1496         # known-to-use DynamicUser=yes services, as setting this system-wide has many undesirable
1497         # side-effects, as it creates its own namespace.
1498         for service in test- systemd-journal-{gatewayd,upload}; do
1499             mkdir -p "$initdir/etc/systemd/system/$service.service.d/"
1500             echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/$service.service.d/99-gcov-rwpaths-override.conf"
1501         done
1502         # Ditto, but for the user daemon
1503         mkdir -p "$initdir/etc/systemd/user/test-.service.d/"
1504         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf"
1505         # Bind the $BUILD_DIR into nspawn containers that are executed using
1506         # machinectl. Unfortunately, the .nspawn files don't support drop-ins
1507         # so we have to inject the bind mount directly into
1508         # the systemd-nspawn@.service unit.
1509         cp "$initdir/usr/lib/systemd/system/systemd-nspawn@.service" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1510         sed -ri "s/^ExecStart=.+$/& --bind=${BUILD_DIR//\//\\\/}/" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1511         # Pass the $BUILD_DIR as $COVERAGE_BUILD_DIR env variable to the system
1512         # manager, similarly to what we do with $ASAN_RT_PATH during sanitized
1513         # builds
1514         mkdir -p "$initdir/etc/systemd/system.conf.d/"
1515         echo -ne "[Manager]\nDefaultEnvironment=COVERAGE_BUILD_DIR=$BUILD_DIR\n" >"$initdir/etc/systemd/system.conf.d/99-COVERAGE_BUILD_DIR.conf"
1516     fi
1517
1518     # If we're built with -Dportabled=false, tests with systemd-analyze
1519     # --profile will fail. Since we need just the profile (text) files, let's
1520     # copy them into the image if they don't exist there.
1521     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1522     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1523         dinfo "Couldn't find portable profiles in the test image"
1524         dinfo "Copying them directly from the source tree"
1525         mkdir -p "$portable_dir"
1526         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1527     fi
1528 }
1529
1530 get_ldpath() {
1531     local rpath
1532     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1533
1534     if [ -z "$rpath" ] ; then
1535         echo "$BUILD_DIR"
1536     else
1537         echo "$rpath"
1538     fi
1539 }
1540
1541 install_missing_libraries() {
1542     dinfo "Install missing libraries"
1543     # install possible missing libraries
1544     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/unit-tests/{,manual/,unsafe/}}*; do
1545         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1546     done
1547
1548     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1549     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1550     # if not present
1551     # See: https://github.com/systemd/systemd/pull/23858
1552     while read -r libgcc_s; do
1553         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1554     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1555
1556     local lib path
1557     # A number of dependencies is now optional via dlopen, so the install
1558     # script will not pick them up, since it looks at linkage.
1559     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1 libarchive; do
1560         ddebug "Searching for $lib via pkg-config"
1561         if pkg-config --exists "$lib"; then
1562                 path="$(pkg-config --variable=libdir "$lib")"
1563                 if [ -z "${path}" ]; then
1564                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1565                     continue
1566                 fi
1567
1568                 if ! [[ ${lib} =~ ^lib ]]; then
1569                         lib="lib${lib}"
1570                 fi
1571                 # p11-kit-1's .so doesn't have the API level in the name
1572                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1573                         lib="libp11-kit"
1574                 fi
1575                 # Some pkg-config files are broken and give out the wrong paths
1576                 # (eg: libcryptsetup), so just ignore them
1577                 inst_libs "${path}/${lib}.so" || true
1578                 inst_library "${path}/${lib}.so" || true
1579
1580                 if [[ "$lib" == "libxkbcommon" ]]; then
1581                     install_x11_keymaps full
1582                 fi
1583         else
1584             ddebug "$lib.pc not found, skipping"
1585             continue
1586         fi
1587     done
1588
1589     # Install extra openssl 3 stuff
1590     path="$(pkg-config --variable=libdir libcrypto)"
1591     inst_simple "${path}/ossl-modules/legacy.so" || true
1592     inst_simple "${path}/ossl-modules/fips.so" || true
1593     inst_simple "${path}/engines-3/afalg.so" || true
1594     inst_simple "${path}/engines-3/capi.so" || true
1595     inst_simple "${path}/engines-3/loader_attic.so" || true
1596     inst_simple "${path}/engines-3/padlock.so" || true
1597
1598     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1599     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1600     # shellcheck disable=SC2046
1601     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1602 }
1603
1604 cleanup_loopdev() {
1605     if [ -n "${LOOPDEV:=}" ]; then
1606         ddebug "losetup -d $LOOPDEV"
1607         losetup -d "${LOOPDEV}"
1608         unset LOOPDEV
1609     fi
1610 }
1611
1612 add_at_exit_handler cleanup_loopdev
1613
1614 create_empty_image() {
1615     if [[ -z "${IMAGE_NAME:=}" ]]; then
1616         echo "create_empty_image: \$IMAGE_NAME not set"
1617         exit 1
1618     fi
1619
1620     # Partition sizes are in MiBs
1621     local root_size=768
1622     local data_size=100
1623     local esp_size=128
1624     local boot_size=128
1625     local total=
1626     if ! get_bool "$NO_BUILD"; then
1627         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1628             root_size=$((root_size + 200))
1629         fi
1630         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1631             root_size=$((root_size + 200))
1632         fi
1633         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1634             root_size=$((root_size + 250))
1635         fi
1636         if get_bool "$IS_BUILT_WITH_ASAN"; then
1637             root_size=$((root_size * 2))
1638         fi
1639     fi
1640
1641     if [[ "${IMAGE_ADDITIONAL_ROOT_SIZE:-0}" -gt 0 ]]; then
1642         root_size=$((root_size + IMAGE_ADDITIONAL_ROOT_SIZE))
1643     fi
1644     if [[ "${IMAGE_ADDITIONAL_DATA_SIZE:-0}" -gt 0 ]]; then
1645         data_size=$((data_size + IMAGE_ADDITIONAL_DATA_SIZE))
1646     fi
1647
1648     total=$((root_size + data_size + esp_size + boot_size))
1649
1650     echo "Setting up ${IMAGE_PUBLIC:?} (${total} MB)"
1651     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1652
1653     # Create the blank file to use as a root filesystem
1654     truncate -s "${total}M" "$IMAGE_PUBLIC"
1655
1656     LOOPDEV="$(losetup --show -P -f "$IMAGE_PUBLIC")"
1657     [[ -b "$LOOPDEV" ]] || return 1
1658     # Create two partitions - a root one and a data one (utilized by some tests)
1659     sfdisk "$LOOPDEV" <<EOF
1660 label: gpt
1661 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B name=esp  size=${esp_size}M
1662 type=$("${SYSTEMD_ID128:?}" show root -Pu) name=root size=${root_size}M bootable
1663 type=BC13C2FF-59E6-4262-A352-B275FD6F7172 name=boot size=${boot_size}M
1664 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1665 EOF
1666
1667     udevadm settle
1668
1669     if ! mkfs -t vfat "${LOOPDEV}p1"; then
1670         dfatal "Failed to mkfs -t vfat ${LOOPDEV}p1"
1671         exit 1
1672     fi
1673
1674     local label=(-L systemd_boot)
1675     # mkfs.reiserfs doesn't know -L. so, use --label instead
1676     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1677     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p2" -q; then
1678         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p2 -q"
1679         exit 1
1680     fi
1681
1682     local label=(-L xbootldr)
1683     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label xbootldr)
1684     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p3" -q; then
1685         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p3 -q"
1686         exit 1
1687     fi
1688 }
1689
1690 mount_initdir() {
1691     if [ -z "${LOOPDEV:=}" ]; then
1692         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1693         LOOPDEV="$(losetup --show -P -f "$image")"
1694         [ -b "$LOOPDEV" ] || return 1
1695
1696         udevadm settle
1697     fi
1698
1699     if ! mountpoint -q "${initdir:?}"; then
1700         mkdir -p "$initdir"
1701         mount "${LOOPDEV}p2" "$initdir"
1702         TEST_SETUP_CLEANUP_ROOTDIR=1
1703     fi
1704 }
1705
1706 cleanup_initdir() {
1707     # only umount if create_empty_image_rootdir() was called to mount it
1708     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1709         _umount_dir "${initdir:?}"
1710     fi
1711 }
1712
1713 umount_loopback() {
1714     # unmount the loopback device from all places. Otherwise we risk file
1715     # system corruption.
1716     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1717         ddebug "Unmounting all uses of $device"
1718         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1719     done
1720 }
1721
1722 create_empty_image_rootdir() {
1723     create_empty_image
1724     mount_initdir
1725 }
1726
1727 check_asan_reports() {
1728     local ret=0
1729     local root="${1:?}"
1730     local log report
1731
1732     if get_bool "$IS_BUILT_WITH_ASAN"; then
1733         ls -l "$root"
1734         if [[ -e "$root/systemd.asan.log.1" ]]; then
1735             cat "$root/systemd.asan.log.1"
1736             ret=$((ret+1))
1737         fi
1738
1739         for log in pid1 journald; do
1740             report="$(find "$root" -name "systemd-$log.*san.log*" -exec cat {} \;)"
1741             if [[ -n "$report" ]]; then
1742                 printf "%s\n" "$report"
1743                 # shellcheck disable=SC2015
1744                 [[ "$log" == journald ]] && cat "$root/systemd-journald.out" || :
1745                 ret=$((ret+1))
1746             fi
1747         done
1748
1749         # May 08 13:23:31 H testleak[2907148]: SUMMARY: AddressSanitizer: 4 byte(s) leaked in 1 allocation(s).
1750         pids="$(
1751             "$JOURNALCTL" -D "$root/var/log/journal" --grep 'SUMMARY: .*Sanitizer:' |
1752                    grep -v -E 'dbus-daemon|dbus-broker-launch' |
1753                    sed -r -n 's/.* .+\[([0-9]+)\]: SUMMARY:.*/\1/p'
1754         )"
1755
1756         if [[ -n "$pids" ]]; then
1757             ret=$((ret+1))
1758             for pid in $pids; do
1759                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1760             done
1761         fi
1762     fi
1763
1764     return $ret
1765 }
1766
1767 check_coverage_reports() {
1768     local root="${1:?}"
1769
1770     if get_bool "$NO_BUILD"; then
1771         return 0
1772     fi
1773     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1774         return 0
1775     fi
1776
1777     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1778         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1779     else
1780         dest="${TESTDIR:?}/coverage-info"
1781     fi
1782
1783     if [[ ! -e "${TESTDIR:?}/coverage-base" ]]; then
1784         # This shouldn't happen, as the report is generated during the setup
1785         # phase (test_setup()).
1786         derror "Missing base coverage report"
1787         return 1
1788     fi
1789
1790     # Create a coverage report that will later be uploaded. Remove info about system
1791     # libraries/headers and generated files, as we don't really care about them.
1792     lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1793     if [[ -f "$dest" ]]; then
1794         # If the destination report file already exists, don't overwrite it, but
1795         # merge it with the already present one - this usually happens when
1796         # running both "parts" of a test in one run (the qemu and the nspawn part).
1797         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1798     else
1799         # If there's no prior coverage report, merge the new one with the base
1800         # report we did during the setup phase (see test_setup()).
1801         lcov --add-tracefile "${TESTDIR:?}/coverage-base" --add-tracefile "${dest}.new" -o "${dest}"
1802     fi
1803     lcov --remove "$dest" -o "$dest" '/usr/include/*' '/usr/lib/*' "${BUILD_DIR:?}/*"
1804     rm -f "${dest}.new"
1805
1806     # If the test logs contain lines like:
1807     #
1808     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1809     #
1810     # it means we're possibly missing some coverage since gcov can't write the stats,
1811     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1812     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1813     # `setfacl` stuff in install_compiled_systemd().
1814     #
1815     # Also, a note: some tests, like TEST-46, overmount /home with tmpfs, which
1816     # means if your build dir is under /home/your-user (which is usually the
1817     # case) you might get bogus errors and missing coverage.
1818     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1819        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1820         derror "Detected possibly missing coverage, check the journal"
1821         return 1
1822     fi
1823
1824     return 0
1825 }
1826
1827 save_journal() {
1828     local source_dir="${1:?}"
1829     local state="${2:?}"
1830     # Default to always saving journal
1831     local save="yes"
1832     local dest_dir dest_name dest
1833
1834     if [[ "${TEST_SAVE_JOURNAL:-}" == "no" ]]; then
1835         save="no"
1836     elif [[ "${TEST_SAVE_JOURNAL:-}" == "fail" && "$state" -eq 0 ]]; then
1837         save="no"
1838     fi
1839
1840     if [[ -n "${ARTIFACT_DIRECTORY:-}" ]]; then
1841         dest_dir="$ARTIFACT_DIRECTORY"
1842         dest_name="${testname:?}.journal"
1843     else
1844         dest_dir="${TESTDIR:?}"
1845         dest_name="system.journal"
1846     fi
1847
1848     # Show messages from the testsuite-XX.service or messages with priority "warning" and higher
1849     echo " --- $source_dir ---"
1850     "$JOURNALCTL" --all --no-pager --no-hostname -o short-monotonic -D "$source_dir" \
1851         _SYSTEMD_UNIT="testsuite-${TESTID:?}.service" + SYSLOG_IDENTIFIER="testsuite-$TESTID.sh" + \
1852         PRIORITY=4 + PRIORITY=3 + PRIORITY=2 + PRIORITY=1 + PRIORITY=0
1853
1854     if get_bool "$save"; then
1855         # If we don't have systemd-journal-remote copy all journals from /var/log/journal/
1856         # to $dest_dir/journals/ as is, otherwise merge all journals into a single .journal
1857         # file
1858         if [[ -z "${SYSTEMD_JOURNAL_REMOTE:-}" ]]; then
1859             dest="$dest_dir/journals"
1860             mkdir -p "$dest"
1861             cp -a "$source_dir/*" "$dest/"
1862         else
1863             dest="$dest_dir/$dest_name"
1864             "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $source_dir"
1865         fi
1866
1867         if [[ -n "${SUDO_USER:-}" ]]; then
1868             setfacl -R -m "user:$SUDO_USER:r-X" "$dest"
1869         fi
1870
1871         # we want to print this sometime later, so save this in a variable
1872         JOURNAL_LIST="$(ls -lR "$dest")"
1873     fi
1874
1875     rm -rf "${source_dir:?}"/*
1876 }
1877
1878 check_result_common() {
1879     local workspace="${1:?}"
1880     local ret
1881
1882     if [ -s "$workspace/failed" ]; then
1883         # Non-empty …/failed has highest priority
1884         cp -a "$workspace/failed" "${TESTDIR:?}/"
1885         if [ -n "${SUDO_USER}" ]; then
1886             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1887         fi
1888         ret=1
1889     elif get_bool "$TIMED_OUT"; then
1890         echo "(timeout)" >"${TESTDIR:?}/failed"
1891         ret=2
1892     elif [ -e "$workspace/testok" ]; then
1893         # …/testok always counts (but with lower priority than …/failed)
1894         ret=0
1895     elif [ -e "$workspace/skipped" ]; then
1896         # …/skipped always counts (a message is expected)
1897         echo "${TESTNAME:?} was skipped:"
1898         cat "$workspace/skipped"
1899         ret=0
1900     else
1901         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1902         ret=3
1903     fi
1904
1905     check_asan_reports "$workspace" || ret=4
1906
1907     check_coverage_reports "$workspace" || ret=5
1908
1909     save_journal "$workspace/var/log/journal" $ret
1910
1911     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1912         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1913     fi
1914
1915     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1916         echo -n "${TESTNAME:?}: "
1917         cat "$TESTDIR/failed"
1918     fi
1919     echo "${JOURNAL_LIST:-"No journals were saved"}"
1920
1921     return ${ret:?}
1922 }
1923
1924 check_result_nspawn() {
1925     local workspace="${1:?}"
1926     local ret=0
1927
1928     # Run a test-specific checks if defined by check_result_nspawn_hook()
1929     if declare -F check_result_nspawn_hook >/dev/null; then
1930         if ! check_result_nspawn_hook "${workspace}"; then
1931             derror "check_result_nspawn_hook() returned with EC > 0"
1932             ret=4
1933         fi
1934     fi
1935
1936     check_result_common "${workspace}" || ret=$?
1937
1938     _umount_dir "${initdir:?}"
1939
1940     return $ret
1941 }
1942
1943 # can be overridden in specific test
1944 check_result_qemu() {
1945     local ret=0
1946     mount_initdir
1947
1948     # Run a test-specific checks if defined by check_result_qemu_hook()
1949     if declare -F check_result_qemu_hook >/dev/null; then
1950         if ! check_result_qemu_hook "${initdir:?}"; then
1951             derror "check_result_qemu_hook() returned with EC > 0"
1952             ret=4
1953         fi
1954     fi
1955
1956     check_result_common "${initdir:?}" || ret=$?
1957
1958     _umount_dir "${initdir:?}"
1959
1960     return $ret
1961 }
1962
1963 check_result_nspawn_unittests() {
1964     local workspace="${1:?}"
1965     local ret=1
1966
1967     [[ -e "$workspace/testok" ]] && ret=0
1968
1969     if [[ -s "$workspace/failed" ]]; then
1970         ret=$((ret + 1))
1971         echo "=== Failed test log ==="
1972         cat "$workspace/failed"
1973     else
1974         if [[ -s "$workspace/skipped" ]]; then
1975             echo "=== Skipped test log =="
1976             cat "$workspace/skipped"
1977             # We might have only skipped tests - that should not fail the job
1978             ret=0
1979         fi
1980         if [[ -s "$workspace/testok" ]]; then
1981             echo "=== Passed tests ==="
1982             cat "$workspace/testok"
1983         fi
1984     fi
1985
1986     get_bool "${TIMED_OUT:=}" && ret=1
1987     check_coverage_reports "$workspace" || ret=5
1988
1989     save_journal "$workspace/var/log/journal" $ret
1990     echo "${JOURNAL_LIST:-"No journals were saved"}"
1991
1992     _umount_dir "${initdir:?}"
1993
1994     return $ret
1995 }
1996
1997 check_result_qemu_unittests() {
1998     local ret=1
1999
2000     mount_initdir
2001     [[ -e "${initdir:?}/testok" ]] && ret=0
2002
2003     if [[ -s "$initdir/failed" ]]; then
2004         ret=$((ret + 1))
2005         echo "=== Failed test log ==="
2006         cat "$initdir/failed"
2007     else
2008         if [[ -s "$initdir/skipped" ]]; then
2009             echo "=== Skipped test log =="
2010             cat "$initdir/skipped"
2011             # We might have only skipped tests - that should not fail the job
2012             ret=0
2013         fi
2014         if [[ -s "$initdir/testok" ]]; then
2015             echo "=== Passed tests ==="
2016             cat "$initdir/testok"
2017         fi
2018     fi
2019
2020     get_bool "${TIMED_OUT:=}" && ret=1
2021     check_coverage_reports "$initdir" || ret=5
2022
2023     save_journal "$initdir/var/log/journal" $ret
2024     echo "${JOURNAL_LIST:-"No journals were saved"}"
2025
2026     _umount_dir "$initdir"
2027
2028     return $ret
2029 }
2030
2031 create_rc_local() {
2032     dinfo "Create rc.local"
2033     mkdir -p "${initdir:?}/etc/rc.d"
2034     cat >"$initdir/etc/rc.d/rc.local" <<EOF
2035 #!/usr/bin/env bash
2036 exit 0
2037 EOF
2038     chmod 0755 "$initdir/etc/rc.d/rc.local"
2039 }
2040
2041 install_execs() {
2042     ddebug "Install executables from the service files"
2043
2044     local pkg_config_path="${BUILD_DIR:?}/src/core/"
2045     local systemunitdir userunitdir exe
2046     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
2047     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
2048     while read -r exe; do
2049         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
2050         # also, plymouth is pulled in by rescue.service, but even there the exit code
2051         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
2052         dinfo "Attempting to install $exe (based on unit file reference)"
2053         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
2054     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
2055 }
2056
2057 generate_module_dependencies() {
2058     dinfo "Generate modules dependencies"
2059     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
2060         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
2061             dfatal "\"depmod -a $KERNEL_VER\" failed."
2062             exit 1
2063     fi
2064 }
2065
2066 install_depmod_files() {
2067     dinfo "Install depmod files"
2068     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
2069     inst "/lib/modules/$KERNEL_VER/modules.builtin"
2070 }
2071
2072 install_plymouth() {
2073     dinfo "Install plymouth"
2074     # install plymouth, if found... else remove plymouth service files
2075     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
2076     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
2077     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
2078     #         image_install plymouth plymouthd
2079     # else
2080         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
2081     # fi
2082 }
2083
2084 install_haveged() {
2085     # If haveged is installed, it's probably included in initrd and needs to be
2086     # installed in the image too.
2087     if [ -x /usr/sbin/haveged ]; then
2088         dinfo "Install haveged files"
2089         inst /usr/sbin/haveged
2090         for u in /usr/lib/systemd/system/haveged*; do
2091             inst "$u"
2092         done
2093     fi
2094 }
2095
2096 install_ld_so_conf() {
2097     dinfo "Install /etc/ld.so.conf*"
2098     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
2099     ldconfig -r "$initdir"
2100 }
2101
2102 install_testuser() {
2103     dinfo "Set up a test user"
2104     # create unprivileged user for user manager tests
2105     mkdir -p "${initdir:?}/etc/sysusers.d"
2106     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
2107 u testuser    4711     "Test User" /home/testuser /bin/bash
2108 EOF
2109
2110     mkdir -p "$initdir/home/testuser"
2111     chmod 0700 "$initdir/home/testuser"
2112     chown 4711:4711 "$initdir/home/testuser"
2113 }
2114
2115 install_config_files() {
2116     dinfo "Install config files"
2117     inst /etc/sysconfig/init || :
2118     inst /etc/passwd
2119     inst /etc/shadow
2120     inst_any /etc/login.defs /usr/etc/login.defs
2121     inst /etc/group
2122     inst /etc/shells
2123     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
2124     inst /etc/pam.conf || :
2125     inst_any /etc/os-release /usr/lib/os-release
2126     inst /etc/localtime
2127     # we want an empty environment
2128     : >"${initdir:?}/etc/environment"
2129     : >"$initdir/etc/machine-id"
2130     : >"$initdir/etc/resolv.conf"
2131
2132     # set the hostname
2133     echo 'H' >"$initdir/etc/hostname"
2134
2135     # let's set up just one image with the traditional verbose output
2136     if [ "${IMAGE_NAME:?}" != "basic" ]; then
2137         mkdir -p "$initdir/etc/systemd/system.conf.d"
2138         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
2139     fi
2140 }
2141
2142 install_basic_tools() {
2143     dinfo "Install basic tools"
2144     image_install "${BASICTOOLS[@]}"
2145     image_install -o sushell
2146     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
2147     image_install -o ldconfig.real
2148 }
2149
2150 install_debug_tools() {
2151     dinfo "Install debug tools"
2152     image_install -o "${DEBUGTOOLS[@]}"
2153
2154     if get_bool "$INTERACTIVE_DEBUG"; then
2155         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
2156         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
2157         mkdir -p "$getty_override"
2158         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
2159         echo 'export TERM=linux' >>"$initdir/etc/profile"
2160
2161         if command -v resize >/dev/null; then
2162             image_install resize
2163             echo "resize" >>"$initdir/etc/profile"
2164         fi
2165
2166         # Sometimes we might end up with plymouthd still running (especially
2167         # with the initrd -> asan_wrapper -> systemd transition), which will eat
2168         # our inputs and make debugging via tty impossible. Let's fix this by
2169         # killing plymouthd explicitly for the interactive sessions.
2170         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
2171         #   - https://github.com/llvm/llvm-project/issues/49223
2172         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
2173         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
2174         cat >"$plymouth_unit" <<EOF
2175 [Unit]
2176 After=multi-user.target
2177
2178 [Service]
2179 ExecStart=sh -c 'killall --verbose plymouthd || :'
2180
2181 [Install]
2182 WantedBy=multi-user.target
2183 EOF
2184         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
2185     fi
2186 }
2187
2188 install_libnss() {
2189     dinfo "Install libnss"
2190     # install libnss_files for login
2191     local NSS_LIBS
2192     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
2193     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
2194         image_install "${NSS_LIBS[@]}"
2195     fi
2196 }
2197
2198 install_dbus() {
2199     dinfo "Install dbus"
2200     inst "${ROOTLIBDIR:?}/system/dbus.socket"
2201
2202     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2203     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
2204         inst "$ROOTLIBDIR/system/dbus-broker.service"
2205         inst /usr/bin/dbus-broker
2206         inst /usr/bin/dbus-broker-launch
2207         image_install -o {/etc,/usr/lib}/systemd/system/dbus.service
2208     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
2209         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2210         inst "$ROOTLIBDIR/system/dbus-daemon.service"
2211         # Alias symlink
2212         image_install -o {/etc,/usr/lib}/systemd/system/dbus.service
2213     else
2214         inst "$ROOTLIBDIR/system/dbus.service"
2215     fi
2216
2217     while read -r file; do
2218         inst "$file"
2219     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
2220
2221     # setup policy for Type=dbus test
2222     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
2223     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
2224 <?xml version="1.0"?>
2225 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2226         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2227 <busconfig>
2228     <policy user="root">
2229         <allow own="systemd.test.ExecStopPost"/>
2230     </policy>
2231 </busconfig>
2232 EOF
2233
2234     # If we run without KVM, bump the service start timeout
2235     if ! get_bool "$QEMU_KVM"; then
2236         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
2237 <?xml version="1.0"?>
2238 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2239         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2240 <busconfig>
2241     <limit name="service_start_timeout">120000</limit>
2242 </busconfig>
2243 EOF
2244         # Bump the client-side timeout in sd-bus as well
2245         mkdir -p "$initdir/etc/systemd/system.conf.d"
2246         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=120' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
2247     fi
2248 }
2249
2250 install_user_dbus() {
2251     dinfo "Install user dbus"
2252     local userunitdir
2253     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
2254         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
2255         userunitdir=/usr/lib/systemd/user
2256     fi
2257
2258     inst "$userunitdir/dbus.socket"
2259     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
2260
2261     # Append the After= dependency on dbus in case it isn't already set up
2262     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
2263     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
2264 [Unit]
2265 After=dbus.service
2266 EOF
2267
2268     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2269     if [ -f "$userunitdir/dbus-broker.service" ]; then
2270         inst "$userunitdir/dbus-broker.service"
2271         image_install -o {/etc,/usr/lib}/systemd/user/dbus.service
2272     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
2273         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2274         inst "$userunitdir/dbus-daemon.service"
2275         # Alias symlink
2276         image_install -o {/etc,/usr/lib}/systemd/user/dbus.service
2277     else
2278         inst "$userunitdir/dbus.service"
2279     fi
2280 }
2281
2282 install_pam() {
2283     dinfo "Install PAM"
2284     local paths=()
2285
2286     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
2287         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
2288     else
2289         paths+=(/lib*/security)
2290     fi
2291
2292     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
2293         [ -d "$d" ] && paths+=("$d")
2294     done
2295
2296     while read -r file; do
2297         inst "$file"
2298     done < <(find "${paths[@]}" -xtype f)
2299
2300     # pam_unix depends on unix_chkpwd.
2301     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2302     image_install -o unix_chkpwd
2303
2304     # set empty root password for easy debugging
2305     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2306
2307     # And make sure pam_unix will accept it by making sure that
2308     # the PAM module has the nullok option.
2309     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2310         [ -d "$initdir/$d" ] || continue
2311         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2312     done
2313 }
2314
2315 install_locales() {
2316     # install only C.UTF-8 and English locales
2317     dinfo "Install locales"
2318
2319     if command -v meson >/dev/null \
2320             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2321             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2322         # locale-gen support
2323         image_install -o locale-gen localedef
2324         inst /etc/locale.gen || :
2325         inst /usr/share/i18n/SUPPORTED || :
2326         inst_recursive /usr/share/i18n/charmaps
2327         inst_recursive /usr/share/i18n/locales
2328         inst_recursive /usr/share/locale/en*
2329         inst_recursive /usr/share/locale/de*
2330         image_install /usr/share/locale/locale.alias
2331         # locale-gen might either generate each locale separately or merge them
2332         # into a single archive
2333         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2334               image_install /usr/lib/locale/locale-archive); then
2335             dfatal "Failed to install required locales"
2336             exit 1
2337         fi
2338     else
2339         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2340     fi
2341 }
2342
2343 # shellcheck disable=SC2120
2344 install_keymaps() {
2345     local i p
2346     local -a prefix=(
2347         "/usr/lib"
2348         "/usr/share"
2349     )
2350
2351     dinfo "Install console keymaps"
2352
2353     if (( $# == 0 )); then
2354         for p in "${prefix[@]}"; do
2355             # The first three paths may be deprecated.
2356             # It seems now the last three paths are used by many distributions.
2357             for i in \
2358                 "$p"/kbd/keymaps/include/* \
2359                 "$p"/kbd/keymaps/i386/include/* \
2360                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2361                 "$p"/kbd/keymaps/legacy/include/* \
2362                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2363                 "$p"/kbd/keymaps/xkb/us*; do
2364                     [[ -f "$i" ]] || continue
2365                     inst "$i"
2366             done
2367         done
2368     else
2369         # When it takes any argument, then install more keymaps.
2370         for p in "${prefix[@]}"; do
2371             for i in \
2372                 "$p"/kbd/keymaps/include/* \
2373                 "$p"/kbd/keymaps/i386/*/* \
2374                 "$p"/kbd/keymaps/legacy/i386/*/* \
2375                 "$p"/kbd/keymaps/xkb/*; do
2376                     [[ -f "$i" ]] || continue
2377                     inst "$i"
2378             done
2379         done
2380     fi
2381 }
2382
2383 install_x11_keymaps() {
2384     dinfo "Install x11 keymaps"
2385
2386     if (( $# == 0 )); then
2387         # Install only keymap list.
2388         inst /usr/share/X11/xkb/rules/base.lst
2389     else
2390         # When it takes any argument, then install all keymaps.
2391         inst_recursive /usr/share/X11/xkb
2392     fi
2393 }
2394
2395 install_zoneinfo() {
2396     dinfo "Install time zones"
2397     inst_any /usr/share/zoneinfo/Asia/Seoul
2398     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2399     inst_any /usr/share/zoneinfo/Australia/Sydney
2400     inst_any /usr/share/zoneinfo/Europe/Berlin
2401     inst_any /usr/share/zoneinfo/Europe/Dublin
2402     inst_any /usr/share/zoneinfo/Europe/Kiev
2403     inst_any /usr/share/zoneinfo/Pacific/Auckland
2404     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2405     inst_any /usr/share/zoneinfo/CET
2406     inst_any /usr/share/zoneinfo/EET
2407     inst_any /usr/share/zoneinfo/UTC
2408 }
2409
2410 install_fonts() {
2411     dinfo "Install system fonts"
2412     for i in \
2413         /usr/{lib,share}/kbd/consolefonts/eurlatgr* \
2414         /usr/{lib,share}/kbd/consolefonts/latarcyrheb-sun16*; do
2415             [[ -f "$i" ]] || continue
2416             inst "$i"
2417     done
2418 }
2419
2420 install_terminfo() {
2421     dinfo "Install terminfo files"
2422     local terminfodir
2423     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2424         [ -f "${terminfodir}/l/linux" ] && break
2425     done
2426     image_install -o "${terminfodir}/l/linux"
2427 }
2428
2429 has_user_dbus_socket() {
2430     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2431         return 0
2432     else
2433         echo "Per-user instances are not supported. Skipping..."
2434         return 1
2435     fi
2436 }
2437
2438 setup_nspawn_root_hook() { :;}
2439
2440 setup_nspawn_root() {
2441     if [ -z "${initdir}" ]; then
2442         dfatal "\$initdir not defined"
2443         exit 1
2444     fi
2445
2446     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2447
2448     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2449         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2450         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2451     fi
2452
2453     setup_nspawn_root_hook
2454 }
2455
2456 setup_basic_dirs() {
2457     mkdir -p "${initdir:?}/run"
2458     mkdir -p "$initdir/etc/systemd/system"
2459     mkdir -p "$initdir/var/log/journal"
2460
2461
2462     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2463         if [ -L "/$d" ]; then
2464             inst_symlink "/$d"
2465         else
2466             inst_dir "/$d"
2467         fi
2468     done
2469
2470     ln -sfn /run "$initdir/var/run"
2471     ln -sfn /run/lock "$initdir/var/lock"
2472 }
2473
2474 mask_supporting_services() {
2475     # mask some services that we do not want to run in these tests
2476     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2477     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2478     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2479     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2480     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2481 }
2482
2483 inst_libs() {
2484     local bin="${1:?}"
2485     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2486     local file line
2487
2488     while read -r line; do
2489         [[ "$line" = 'not a dynamic executable' ]] && break
2490         # Ignore errors about our own stuff missing. This is most likely caused
2491         # by ldd attempting to use the unprefixed RPATH.
2492         [[ "$line" =~ (libsystemd|libudev).*\ not\ found ]] && continue
2493
2494         if [[ "$line" =~ not\ found ]]; then
2495             dfatal "Missing a shared library required by $bin."
2496             dfatal "Run \"ldd $bin\" to find out what it is."
2497             dfatal "$line"
2498             dfatal "Cannot create a test image."
2499             exit 1
2500         fi
2501
2502         if [[ "$line" =~ $so_regex ]]; then
2503             file="${BASH_REMATCH[1]}"
2504             [[ -e "${initdir:?}/$file" ]] && continue
2505             inst_library "$file"
2506         fi
2507     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2508 }
2509
2510 import_testdir() {
2511     # make sure we don't get a stale LOOPDEV value from old times
2512     local _LOOPDEV="${LOOPDEV:=}"
2513     # We don't want shellcheck to follow & check the $STATEFILE
2514     # shellcheck source=/dev/null
2515     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2516     LOOPDEV="$_LOOPDEV"
2517     if [[ ! -d "$TESTDIR" ]]; then
2518         if [[ -z "$TESTDIR" ]]; then
2519             TESTDIR="$(mktemp --tmpdir=$WORKDIR -d -t systemd-test.XXXXXX)"
2520         else
2521             mkdir -p "$TESTDIR"
2522         fi
2523
2524         cat >"$STATEFILE" <<EOF
2525 TESTDIR="$TESTDIR"
2526 EOF
2527         export TESTDIR
2528     fi
2529
2530     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2531     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2532 }
2533
2534 import_initdir() {
2535     initdir="${TESTDIR:?}/root"
2536     mkdir -p "$initdir"
2537     export initdir
2538 }
2539
2540 get_cgroup_hierarchy() {
2541     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2542         cgroup2fs)
2543             echo "unified"
2544             ;;
2545         tmpfs)
2546             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2547                 echo "hybrid"
2548             else
2549                 echo "legacy"
2550             fi
2551             ;;
2552         *)
2553             dfatal "Failed to determine host's cgroup hierarchy"
2554             exit 1
2555     esac
2556 }
2557
2558 ## @brief Converts numeric logging level to the first letter of level name.
2559 #
2560 # @param lvl Numeric logging level in range from 1 to 6.
2561 # @retval 1 if @a lvl is out of range.
2562 # @retval 0 if @a lvl is correct.
2563 # @result Echoes first letter of level name.
2564 _lvl2char() {
2565     case "$1" in
2566         1) echo F;;
2567         2) echo E;;
2568         3) echo W;;
2569         4) echo I;;
2570         5) echo D;;
2571         6) echo T;;
2572         *) return 1;;
2573     esac
2574 }
2575
2576 ## @brief Internal helper function for _do_dlog()
2577 #
2578 # @param lvl Numeric logging level.
2579 # @param msg Message.
2580 # @retval 0 It's always returned, even if logging failed.
2581 #
2582 # @note This function is not supposed to be called manually. Please use
2583 # dtrace(), ddebug(), or others instead which wrap this one.
2584 #
2585 # This function calls _do_dlog() either with parameter msg, or if
2586 # none is given, it will read standard input and will use every line as
2587 # a message.
2588 #
2589 # This enables:
2590 # dwarn "This is a warning"
2591 # echo "This is a warning" | dwarn
2592 LOG_LEVEL="${LOG_LEVEL:-4}"
2593
2594 dlog() {
2595     local lvl lvlc
2596
2597     [ -z "$LOG_LEVEL" ] && return 0
2598     lvl="${1:?}"; shift
2599     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2600     lvlc="$(_lvl2char "$lvl")" || return 0
2601
2602     if [ $# -ge 1 ]; then
2603         echo "$lvlc: $*"
2604     else
2605         while read -r line; do
2606             echo "$lvlc: " "$line"
2607         done
2608     fi
2609 }
2610
2611 ## @brief Logs message at TRACE level (6)
2612 #
2613 # @param msg Message.
2614 # @retval 0 It's always returned, even if logging failed.
2615 dtrace() {
2616     set +x
2617     dlog 6 "$@"
2618     if get_bool "${debug:=}"; then
2619         set -x
2620     fi
2621 }
2622
2623 ## @brief Logs message at DEBUG level (5)
2624 #
2625 # @param msg Message.
2626 # @retval 0 It's always returned, even if logging failed.
2627 ddebug() {
2628     dlog 5 "$@"
2629 }
2630
2631 ## @brief Logs message at INFO level (4)
2632 #
2633 # @param msg Message.
2634 # @retval 0 It's always returned, even if logging failed.
2635 dinfo() {
2636     set +x
2637     dlog 4 "$@"
2638     if get_bool "${debug:=}"; then
2639         set -x
2640     fi
2641 }
2642
2643 ## @brief Logs message at WARN level (3)
2644 #
2645 # @param msg Message.
2646 # @retval 0 It's always returned, even if logging failed.
2647 dwarn() {
2648     set +x
2649     dlog 3 "$@"
2650     if get_bool "${debug:=}"; then
2651         set -x
2652     fi
2653 }
2654
2655 ## @brief Logs message at ERROR level (2)
2656 #
2657 # @param msg Message.
2658 # @retval 0 It's always returned, even if logging failed.
2659 derror() {
2660     dlog 2 "$@"
2661 }
2662
2663 ## @brief Logs message at FATAL level (1)
2664 #
2665 # @param msg Message.
2666 # @retval 0 It's always returned, even if logging failed.
2667 dfatal() {
2668     set +x
2669     dlog 1 "$@"
2670     if get_bool "${debug:=}"; then
2671         set -x
2672     fi
2673 }
2674
2675
2676 # Generic substring function.  If $2 is in $1, return 0.
2677 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2678
2679 # normalize_path <path>
2680 # Prints the normalized path, where it removes any duplicated
2681 # and trailing slashes.
2682 # Example:
2683 # $ normalize_path ///test/test//
2684 # /test/test
2685 normalize_path() {
2686     shopt -q -s extglob
2687     set -- "${1//+(\/)//}"
2688     shopt -q -u extglob
2689     echo "${1%/}"
2690 }
2691
2692 # convert_abs_rel <from> <to>
2693 # Prints the relative path, when creating a symlink to <to> from <from>.
2694 # Example:
2695 # $ convert_abs_rel /usr/bin/test /bin/test-2
2696 # ../../bin/test-2
2697 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2698 convert_abs_rel() {
2699     local __current __absolute __abssize __cursize __newpath
2700     local -i __i __level
2701
2702     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2703
2704     # corner case #1 - self looping link
2705     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2706
2707     # corner case #2 - own dir link
2708     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2709
2710     IFS="/" read -ra __current <<< "$1"
2711     IFS="/" read -ra __absolute <<< "$2"
2712
2713     __abssize=${#__absolute[@]}
2714     __cursize=${#__current[@]}
2715
2716     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2717         (( __level++ ))
2718         if (( __level > __abssize || __level > __cursize ))
2719         then
2720             break
2721         fi
2722     done
2723
2724     for ((__i = __level; __i < __cursize-1; __i++)); do
2725         if ((__i > __level))
2726         then
2727             __newpath=$__newpath"/"
2728         fi
2729         __newpath=$__newpath".."
2730     done
2731
2732     for ((__i = __level; __i < __abssize; __i++)); do
2733         if [[ -n $__newpath ]]
2734         then
2735             __newpath=$__newpath"/"
2736         fi
2737         __newpath=$__newpath${__absolute[__i]}
2738     done
2739
2740     echo "$__newpath"
2741 }
2742
2743
2744 # Install a directory, keeping symlinks as on the original system.
2745 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2746 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2747 # and a symlink ${initdir}/lib -> lib64.
2748 inst_dir() {
2749     local dir="${1:?}"
2750     local part="${dir%/*}"
2751     local file
2752
2753     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2754
2755     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2756         dir="$part $dir"
2757         part="${part%/*}"
2758     done
2759
2760     # iterate over parent directories
2761     for file in $dir; do
2762         [[ -e "${initdir}/$file" ]] && continue
2763         if [[ -L $file ]]; then
2764             inst_symlink "$file"
2765         else
2766             # create directory
2767             mkdir -m 0755 "${initdir}/$file" || return 1
2768             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2769             chmod u+w "${initdir}/$file"
2770         fi
2771     done
2772 }
2773
2774 # $1 = file to copy to ramdisk
2775 # $2 (optional) Name for the file on the ramdisk
2776 # Location of the image dir is assumed to be $initdir
2777 # We never overwrite the target if it exists.
2778 inst_simple() {
2779     [[ -f "${1:?}" ]] || return 1
2780     strstr "$1" "/" || return 1
2781
2782     local src="$1"
2783     local target="${2:-$1}"
2784     if ! [[ -d ${initdir:?}/$target ]]; then
2785         [[ -e ${initdir}/$target ]] && return 0
2786         [[ -L ${initdir}/$target ]] && return 0
2787         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2788     fi
2789     # install checksum files also
2790     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2791         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2792     fi
2793     ddebug "Installing $src"
2794     cp --sparse=always --force --dereference --preserve=all "$src" "${initdir}/$target"
2795 }
2796
2797 # find symlinks linked to given library file
2798 # $1 = library file
2799 # Function searches for symlinks by stripping version numbers appended to
2800 # library filename, checks if it points to the same target and finally
2801 # prints the list of symlinks to stdout.
2802 #
2803 # Example:
2804 # rev_lib_symlinks libfoo.so.8.1
2805 # output: libfoo.so.8 libfoo.so
2806 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2807 rev_lib_symlinks() {
2808     local fn="${1:?}"
2809     local links=""
2810     local orig
2811     orig="$(readlink -f "$1")"
2812
2813     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2814
2815     until [[ "${fn##*.}" == so ]]; do
2816         fn="${fn%.*}"
2817         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2818     done
2819
2820     echo "${links}"
2821 }
2822
2823 # Same as above, but specialized to handle dynamic libraries.
2824 # It handles making symlinks according to how the original library
2825 # is referenced.
2826 inst_library() {
2827     local src="${1:?}"
2828     local dest="${2:-$1}"
2829     local reallib symlink
2830
2831     strstr "$1" "/" || return 1
2832     [[ -e ${initdir:?}/$dest ]] && return 0
2833     if [[ -L $src ]]; then
2834         # install checksum files also
2835         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2836             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2837         fi
2838         reallib="$(readlink -f "$src")"
2839         inst_simple "$reallib" "$reallib"
2840         inst_dir "${dest%/*}"
2841         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2842         ddebug "Creating symlink $reallib -> $dest"
2843         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2844     else
2845         inst_simple "$src" "$dest"
2846     fi
2847
2848     # Create additional symlinks.  See rev_symlinks description.
2849     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2850         if [[ ! -e "$initdir/$symlink" ]]; then
2851             ddebug "Creating extra symlink: $symlink"
2852             inst_symlink "$symlink"
2853         fi
2854     done
2855 }
2856
2857 # find a binary.  If we were not passed the full path directly,
2858 # search in the usual places to find the binary.
2859 find_binary() {
2860     local bin="${1:?}"
2861     if [[ -z ${bin##/*} ]]; then
2862         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2863             echo "$bin"
2864             return 0
2865         fi
2866     fi
2867
2868     type -P "$bin"
2869 }
2870
2871 # Same as above, but specialized to install binary executables.
2872 # Install binary executable, and all shared library dependencies, if any.
2873 inst_binary() {
2874     local bin="${1:?}"
2875     local path target
2876
2877     # In certain cases we might attempt to install a binary which is already
2878     # present in the test image, yet it's missing from the host system.
2879     # In such cases, let's check if the binary indeed exists in the image
2880     # before doing any other checks. If it does, immediately return with
2881     # success.
2882     if [[ $# -eq 1 ]]; then
2883         for path in "" bin sbin usr/bin usr/sbin; do
2884             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2885         done
2886     fi
2887
2888     bin="$(find_binary "$bin")" || return 1
2889     target="${2:-$bin}"
2890     [[ -e "${initdir:?}/$target" ]] && return 0
2891     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2892
2893     local file line
2894     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2895     # DSOs provided by systemd
2896     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2897     local wrap_binary=0
2898     local enable_lsan=0
2899     # I love bash!
2900     while read -r line; do
2901         [[ "$line" = 'not a dynamic executable' ]] && break
2902
2903         # Ignore errors about our own stuff missing. This is most likely caused
2904         # by ldd attempting to use the unprefixed RPATH.
2905         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2906
2907         # We're built with ASan and the target binary loads one of the systemd's
2908         # DSOs, so we need to tweak the environment before executing the binary
2909         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2910             wrap_binary=1
2911         fi
2912
2913         if [[ "$line" =~ $so_regex ]]; then
2914             file="${BASH_REMATCH[1]}"
2915             [[ -e "${initdir}/$file" ]] && continue
2916             inst_library "$file"
2917             continue
2918         fi
2919
2920         if [[ "$line" =~ not\ found ]]; then
2921             dfatal "Missing a shared library required by $bin."
2922             dfatal "Run \"ldd $bin\" to find out what it is."
2923             dfatal "$line"
2924             dfatal "Cannot create a test image."
2925             exit 1
2926         fi
2927     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2928
2929     # Same as above, but we need to wrap certain libraries unconditionally
2930     #
2931     # chgrp, chown, getent, login, setfacl, su, useradd, userdel
2932     #    - dlopen() (not only) systemd's PAM modules
2933     # ls, mkfs.*, mksquashfs, mkswap, setpriv, stat
2934     #   - pull in nss_systemd with certain options (like ls -l) when
2935     #     nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2936     # delv, dig - pull in nss_resolve if `resolve` is in nsswitch.conf
2937     # tar - called by machinectl in TEST-25
2938     bin_rx='/(agetty|chgrp|chown|curl|delv|dig|getfacl|getent|id|login|ls|mkfs\.[a-z0-9]+|mksquashfs|mkswap|setfacl|setpriv|stat|su|tar|useradd|userdel)$'
2939     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ $bin_rx ]]; then
2940         wrap_binary=1
2941         # Ugh, so we want to disable LSan in most cases for the wrapped binaries, since
2942         # we don't care about memory leaks in such binaries. However, in certain cases
2943         # the external binary is the only interface for the systemd code, like for
2944         # the systemd NSS modules, where we want to detect memory leaks. So let's
2945         # do another check to decide if we want to enable LSan for given binary.
2946         if [[ "$bin" =~ /getent$ ]]; then
2947             enable_lsan=1
2948         fi
2949     fi
2950
2951     # If the target binary is built with ASan support, we don't need to wrap
2952     # it, as it should handle everything by itself
2953     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2954         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2955         # Install the target binary with a ".orig" suffix
2956         inst_simple "$bin" "${target}.orig"
2957         # Create a simple shell wrapper in place of the target binary, which
2958         # sets necessary ASan-related env variables and then exec()s the
2959         # suffixed target binary
2960         cat >"$initdir/$target" <<EOF
2961 #!/bin/bash
2962 # Preload the ASan runtime DSO, otherwise ASAn will complain
2963 export LD_PRELOAD="$ASAN_RT_PATH"
2964 # Disable LSan to speed things up, since we don't care about leak reports
2965 # from 'external' binaries
2966 export ASAN_OPTIONS=detect_leaks=$enable_lsan
2967 # Set argv[0] to the original binary name without the ".orig" suffix
2968 exec -a "\$0" -- "${target}.orig" "\$@"
2969 EOF
2970         chmod +x "$initdir/$target"
2971     else
2972         inst_simple "$bin" "$target"
2973     fi
2974 }
2975
2976 # same as above, except for shell scripts.
2977 # If your shell script does not start with shebang, it is not a shell script.
2978 inst_script() {
2979     local bin line shebang_regex
2980     bin="$(find_binary "${1:?}")" || return 1
2981     shift
2982
2983     read -r -n 80 line <"$bin"
2984     # If debug is set, clean unprintable chars to prevent messing up the term
2985     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2986     shebang_regex='(#! *)(/[^ ]+).*'
2987     [[ "$line" =~ $shebang_regex ]] || return 1
2988     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2989 }
2990
2991 # same as above, but specialized for symlinks
2992 inst_symlink() {
2993     local src="${1:?}"
2994     local target="${2:-$src}"
2995     local realsrc
2996
2997     strstr "$src" "/" || return 1
2998     [[ -L "$src" ]] || return 1
2999     [[ -L "${initdir:?}/$target" ]] && return 0
3000     realsrc="$(readlink -f "$src")"
3001     if ! [[ -e "$initdir/$realsrc" ]]; then
3002         if [[ -d "$realsrc" ]]; then
3003             inst_dir "$realsrc"
3004         else
3005             inst "$realsrc"
3006         fi
3007     fi
3008     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
3009     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
3010     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
3011 }
3012
3013 # attempt to install any programs specified in a udev rule
3014 inst_rule_programs() {
3015     local rule="${1:?}"
3016     local prog bin
3017
3018     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
3019         if [ -x "/lib/udev/$prog" ]; then
3020             bin="/lib/udev/$prog"
3021         else
3022             if ! bin="$(find_binary "$prog")"; then
3023                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
3024                 continue
3025             fi
3026         fi
3027
3028         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
3029         image_install "$bin"
3030     done
3031 }
3032
3033 # udev rules always get installed in the same place, so
3034 # create a function to install them to make life simpler.
3035 inst_rules() {
3036     local target=/etc/udev/rules.d
3037     local found rule
3038
3039     inst_dir "/lib/udev/rules.d"
3040     inst_dir "$target"
3041     for rule in "$@"; do
3042         if [ "${rule#/}" = "$rule" ]; then
3043             for r in /lib/udev/rules.d /etc/udev/rules.d; do
3044                 if [[ -f "$r/$rule" ]]; then
3045                     found="$r/$rule"
3046                     inst_simple "$found"
3047                     inst_rule_programs "$found"
3048                 fi
3049             done
3050         fi
3051         for r in '' ./; do
3052             if [[ -f "${r}${rule}" ]]; then
3053                 found="${r}${rule}"
3054                 inst_simple "$found" "$target/${found##*/}"
3055                 inst_rule_programs "$found"
3056             fi
3057         done
3058         [[ $found ]] || dinfo "Skipping udev rule: $rule"
3059         found=
3060     done
3061 }
3062
3063 # general purpose installation function
3064 # Same args as above.
3065 inst() {
3066     case $# in
3067         1) ;;
3068         2)
3069             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
3070             [[ "$initdir" = "$2" ]] && set "$1"
3071             ;;
3072         3)
3073             [[ -z "$initdir" ]] && export initdir="$2"
3074             set "$1" "$3"
3075             ;;
3076         *)
3077             dfatal "inst only takes 1 or 2 or 3 arguments"
3078             exit 1
3079             ;;
3080     esac
3081
3082     local fun
3083     for fun in inst_symlink inst_script inst_binary inst_simple; do
3084         "$fun" "$@" && return 0
3085     done
3086
3087     dwarn "Failed to install '$1'"
3088     return 1
3089 }
3090
3091 # install any of listed files
3092 #
3093 # If first argument is '-d' and second some destination path, first accessible
3094 # source is installed into this path, otherwise it will installed in the same
3095 # path as source.  If none of listed files was installed, function return 1.
3096 # On first successful installation it returns with 0 status.
3097 #
3098 # Example:
3099 #
3100 # inst_any -d /bin/foo /bin/bar /bin/baz
3101 #
3102 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
3103 # initrd.
3104 inst_any() {
3105     local dest file
3106
3107     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
3108
3109     for file in "$@"; do
3110         if [[ -e "$file" ]]; then
3111             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
3112             inst "$file" && return 0
3113         fi
3114     done
3115
3116     return 1
3117 }
3118
3119 inst_recursive() {
3120     local p item
3121
3122     for p in "$@"; do
3123         # Make sure the source exists, as the process substitution below
3124         # suppresses errors
3125         stat "$p" >/dev/null || return 1
3126
3127         while read -r item; do
3128             if [[ -d "$item" ]]; then
3129                 inst_dir "$item"
3130             elif [[ -f "$item" ]]; then
3131                 inst_simple "$item"
3132             fi
3133         done < <(find "$p" 2>/dev/null)
3134     done
3135 }
3136
3137 # image_install [-o ] <file> [<file> ... ]
3138 # Install <file> to the test image
3139 # -o optionally install the <file> and don't fail, if it is not there
3140 image_install() {
3141     local optional=no
3142     local prog="${1:?}"
3143
3144     if [[ "$prog" = '-o' ]]; then
3145         optional=yes
3146         shift
3147     fi
3148
3149     for prog in "$@"; do
3150         if ! inst "$prog" ; then
3151             if get_bool "$optional"; then
3152                 dinfo "Skipping program $prog as it cannot be found and is" \
3153                     "flagged to be optional"
3154             else
3155                 dfatal "Failed to install $prog"
3156                 exit 1
3157             fi
3158         fi
3159     done
3160 }
3161
3162 # Install a single kernel module along with any firmware it may require.
3163 # $1 = full path to kernel module to install
3164 install_kmod_with_fw() {
3165     local module="${1:?}"
3166     # no need to go further if the module is already installed
3167     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
3168     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
3169
3170     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
3171
3172     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
3173
3174     local modname="${module##*/}"
3175     local fwdir found fw
3176     modname="${modname%.ko*}"
3177
3178     while read -r fw; do
3179         found=
3180         for fwdir in /lib/firmware/updates /lib/firmware; do
3181             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
3182                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
3183                 found=yes
3184             fi
3185         done
3186         if ! get_bool "$found"; then
3187             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
3188                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
3189                     "\"${modname}.ko\""
3190             else
3191                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
3192                     "\"${modname}.ko\""
3193             fi
3194         fi
3195     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
3196     return 0
3197 }
3198
3199 # Do something with all the dependencies of a kernel module.
3200 # Note that kernel modules depend on themselves using the technique we use
3201 # $1 = function to call for each dependency we find
3202 #      It will be passed the full path to the found kernel module
3203 # $2 = module to get dependencies for
3204 # rest of args = arguments to modprobe
3205 for_each_kmod_dep() {
3206     local func="${1:?}"
3207     local kmod="${2:?}"
3208     local found=0
3209     local cmd modpath
3210     shift 2
3211
3212     while read -r cmd modpath _; do
3213         [[ "$cmd" = insmod ]] || continue
3214         "$func" "$modpath" || return $?
3215         found=1
3216     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
3217
3218     ! get_bool "$found" && return 1
3219     return 0
3220 }
3221
3222 # instmods [-c] <kernel module> [<kernel module> ... ]
3223 # instmods [-c] <kernel subsystem>
3224 # install kernel modules along with all their dependencies.
3225 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
3226 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
3227 instmods() {
3228     local check=no
3229     if [[ $# -ge 0 && "$1" = '-c' ]]; then
3230         check=yes
3231         shift
3232     fi
3233
3234     inst1mod() {
3235         local mod="${1:?}"
3236         local ret=0
3237         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
3238
3239         case "$mod" in
3240             =*)
3241                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
3242                     (
3243                         [[ "$mpargs" ]] && echo "$mpargs"
3244                         cat "${mod_dir}/modules.${mod#=}"
3245                     ) | instmods
3246                 else
3247                     (
3248                         [[ "$mpargs" ]] && echo "$mpargs"
3249                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
3250                     ) | instmods
3251                 fi
3252                 ;;
3253             --*)
3254                 mpargs+=" $mod"
3255                 ;;
3256             i2o_scsi)
3257                 # Do not load this diagnostic-only module
3258                 return
3259                 ;;
3260             *)
3261                 mod=${mod##*/}
3262                 # if we are already installed, skip this module and go on
3263                 # to the next one.
3264                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
3265
3266                 # We use '-d' option in modprobe only if modules prefix path
3267                 # differs from default '/'.  This allows us to use Dracut with
3268                 # old version of modprobe which doesn't have '-d' option.
3269                 local mod_dirname=${mod_dir%%/lib/modules/*}
3270                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
3271
3272                 # ok, load the module, all its dependencies, and any firmware
3273                 # it may require
3274                 for_each_kmod_dep install_kmod_with_fw "$mod" \
3275                     --set-version "$KERNEL_VER" \
3276                     ${mod_dirname:+"$mod_dirname"} \
3277                     ${mpargs:+"$mpargs"}
3278                 ((ret+=$?))
3279                 ;;
3280         esac
3281         return "$ret"
3282     }
3283
3284     local mod mpargs
3285
3286     if [[ $# -eq 0 ]]; then  # filenames from stdin
3287         while read -r mod; do
3288             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3289                 dfatal "Failed to install $mod"
3290                 return 1
3291             fi
3292         done
3293     fi
3294
3295     for mod in "$@"; do # filenames as arguments
3296         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3297             dfatal "Failed to install $mod"
3298             return 1
3299         fi
3300     done
3301
3302     return 0
3303 }
3304
3305 _umount_dir() {
3306     local mountpoint="${1:?}"
3307     if mountpoint -q "$mountpoint"; then
3308         ddebug "umount $mountpoint"
3309         umount "$mountpoint"
3310     fi
3311 }
3312
3313 # can be overridden in specific test
3314 test_setup_cleanup() {
3315     cleanup_initdir
3316 }
3317
3318 _test_cleanup() {
3319     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3320     (
3321         set +e
3322         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3323         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3324         # If multiple setups/cleans are ran in parallel, this can cause a race
3325         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3326             rm -vf "${IMAGESTATEDIR}/default.img"
3327         fi
3328         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3329         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3330     ) || :
3331 }
3332
3333 # can be overridden in specific test
3334 test_cleanup() {
3335     _test_cleanup
3336 }
3337
3338 test_cleanup_again() {
3339     [ -n "$TESTDIR" ] || return
3340     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3341     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3342 }
3343
3344 test_create_image() {
3345     create_empty_image_rootdir
3346
3347     # Create what will eventually be our root filesystem onto an overlay
3348     (
3349         LOG_LEVEL=5
3350         setup_basic_environment
3351     )
3352 }
3353
3354 test_setup() {
3355     if ! get_bool "$NO_BUILD" && \
3356          get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3357          command -v meson >/dev/null && \
3358          [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3359         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3360         exit 1
3361     fi
3362
3363     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3364         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3365         mount_initdir
3366     else
3367         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3368             # default.img is the base that every test uses and optionally appends to
3369             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3370                 # Create the backing public image, but then completely unmount
3371                 # it and drop the loopback device responsible for it, since we're
3372                 # going to symlink/copy the image and mount it again from
3373                 # elsewhere.
3374                 local image_old="${IMAGE_PUBLIC}"
3375                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3376                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3377                 fi
3378                 test_create_image
3379                 test_setup_cleanup
3380                 umount_loopback
3381                 cleanup_loopdev
3382                 IMAGE_PUBLIC="${image_old}"
3383             fi
3384             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3385                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3386             fi
3387         fi
3388
3389         local hook_defined
3390         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3391
3392         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3393         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3394             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3395         else
3396             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3397         fi
3398
3399         mount_initdir
3400
3401         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3402             dinfo "Masking supporting services"
3403             mask_supporting_services
3404         fi
3405
3406         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
3407             # Do an initial coverage capture, to make sure the final report includes
3408             # files that the tests didn't touch at all
3409             lcov --initial --capture --directory "${initdir}/${BUILD_DIR:?}" --output-file "${TESTDIR:?}/coverage-base"
3410         fi
3411
3412         if get_bool "$hook_defined"; then
3413             test_append_files "${initdir:?}"
3414         fi
3415     fi
3416
3417     setup_nspawn_root
3418 }
3419
3420 test_run() {
3421     local test_id="${1:?}"
3422     mount_initdir
3423
3424     if ! get_bool "${TEST_NO_QEMU:=}"; then
3425         if run_qemu "$test_id"; then
3426             check_result_qemu || { echo "qemu test failed"; return 1; }
3427         else
3428             dwarn "can't run qemu, skipping"
3429         fi
3430     fi
3431     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3432         mount_initdir
3433         if run_nspawn "${initdir:?}" "$test_id"; then
3434             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3435         else
3436             dwarn "can't run systemd-nspawn, skipping"
3437         fi
3438
3439         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3440             dir="$TESTDIR/unprivileged-nspawn-root"
3441             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3442                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3443             else
3444                 dwarn "can't run systemd-nspawn, skipping"
3445             fi
3446         fi
3447     fi
3448     return 0
3449 }
3450
3451 do_test() {
3452     if [[ $UID != "0" ]]; then
3453         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3454         exit 0
3455     fi
3456
3457     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3458         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3459         exit 0
3460     fi
3461
3462     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3463         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3464         exit 0
3465     fi
3466
3467     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3468         TEST_NO_QEMU=1
3469     fi
3470
3471     # Detect lib paths
3472     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3473         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3474     done
3475
3476     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3477         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3478     done
3479
3480     mkdir -p "$WORKDIR"
3481     mkdir -p "$STATEDIR"
3482
3483     import_testdir
3484     import_initdir
3485
3486     if [ -n "${SUDO_USER}" ]; then
3487         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3488         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3489     fi
3490
3491     testname="$(basename "$PWD")"
3492
3493     while (($# > 0)); do
3494         case $1 in
3495             --run)
3496                 echo "${testname} RUN: $TEST_DESCRIPTION"
3497                 test_run "$TESTID"
3498                 ret=$?
3499                 if [ $ret -eq 0 ]; then
3500                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3501                 else
3502                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3503                 fi
3504                 exit $ret
3505                 ;;
3506             --setup)
3507                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3508                 test_setup
3509                 test_setup_cleanup
3510                 ;;
3511             --clean)
3512                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3513                 test_cleanup
3514                 ;;
3515             --clean-again)
3516                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3517                 test_cleanup_again
3518                 ;;
3519             --all)
3520                 ret=0
3521                 echo -n "${testname}: $TEST_DESCRIPTION "
3522                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3523                 # and loop devices will leak
3524                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3525                 if [ $ret -eq 0 ]; then
3526                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3527                 fi
3528                 if [ $ret -eq 0 ]; then
3529                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3530                 fi
3531                 test_cleanup
3532                 if [ $ret -eq 0 ]; then
3533                     # $TESTLOG is in $STATEDIR, so clean it up only on success
3534                     [[ -n "$STATEDIR" ]] && rm -vfr "$STATEDIR"
3535                     echo "[OK]"
3536                 else
3537                     echo "[FAILED]"
3538                     echo "see $TESTLOG"
3539                 fi
3540                 exit $ret
3541                 ;;
3542             *)
3543                 break
3544                 ;;
3545         esac
3546         shift
3547     done
3548 }