test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 # Simple wrapper to unify boolean checks.
  18 # Note: this function needs to stay near the top of the file, so we can use it
  19 #       in code in the outermost scope.
  20 get_bool() {
  21     # Make the value lowercase to make the regex matching simpler
  22     local _bool="${1,,}"
  23
  24     # Consider empty value as "false"
  25     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  26         return 1
  27     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  28         return 0
  29     else
  30         echo >&2 "Value '$_bool' is not a valid boolean value"
  31         exit 1
  32     fi
  33 }
  34
  35 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  36 export PATH
  37
  38 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  39 # shellcheck source=/dev/null
  40 source "$os_release"
  41 [[ "$ID" == "debian" || " $ID_LIKE " == *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=no
  42 # shellcheck disable=SC2034
  43 [[ "$ID" == "ubuntu" ]] && LOOKS_LIKE_UBUNTU=yes || LOOKS_LIKE_UBUNTU=no
  44 [[ "$ID" == "arch" || " $ID_LIKE " == *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=no
  45 [[ "$ID" == "fedora" ]] && LOOKS_LIKE_FEDORA=yes || LOOKS_LIKE_FEDORA=no
  46 [[ " $ID_LIKE " == *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=no
  47
  48 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  49 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  50 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  51 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  52 get_bool "$LOOKS_LIKE_SUSE" && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  53 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  54 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  55 # Note that defining a different IMAGE_NAME in a test setup script will only result
  56 # in default.img being copied and renamed. It can then be extended by defining
  57 # a test_append_files() function. The $1 parameter will be the root directory.
  58 # To force creating a new image from scratch (eg: to encrypt it), also define
  59 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  60 IMAGE_NAME=${IMAGE_NAME:-default}
  61 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  62 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  63 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  64 LOOPDEV=
  65
  66 # Since in Bash we can have only one handler per signal, let's overcome this
  67 # limitation by having one global handler for the EXIT signal which executes
  68 # all registered handlers
  69 _AT_EXIT_HANDLERS=()
  70 _at_exit() {
  71     set +e
  72
  73     # Run the EXIT handlers in reverse order
  74     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  75         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  76         eval "${_AT_EXIT_HANDLERS[$i]}"
  77     done
  78 }
  79
  80 trap _at_exit EXIT
  81
  82 add_at_exit_handler() {
  83     _AT_EXIT_HANDLERS+=("${1:?}")
  84 }
  85
  86 # Decide if we can (and want to) run qemu with KVM acceleration.
  87 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  88 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  89 # running under KVM. If these conditions are met, enable KVM (and possibly
  90 # nested KVM), otherwise disable it.
  91 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  92     QEMU_KVM=yes
  93 else
  94     QEMU_KVM=no
  95 fi
  96
  97 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  98     echo "WARNING! Cannot determine libdir from pkg-config, assuming /usr/lib/systemd" >&2
  99     ROOTLIBDIR=/usr/lib/systemd
 100 fi
 101
 102 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
 103 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 104 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 105 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 106 # These variables are used by test scripts
 107 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR
 108
 109 TOOLS_DIR="$SOURCE_DIR/tools"
 110 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 111 if get_bool "${NO_BUILD:=}"; then
 112     BUILD_DIR="$SOURCE_DIR"
 113 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 114     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 115     exit 1
 116 fi
 117
 118 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 119 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 120 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 121 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 122 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 123 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 124 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 125 SYSTEMD_ID128="${SYSTEMD_ID128:-$(command -v "$BUILD_DIR/systemd-id128" || command -v systemd-id128)}"
 126
 127 TESTFILE="${BASH_SOURCE[1]}"
 128 if [ -z "$TESTFILE" ]; then
 129     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 130     exit 1
 131 fi
 132 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 133
 134 WORKDIR="/var/tmp/systemd-tests"
 135 if get_bool "${NO_BUILD:=}"; then
 136     STATEDIR="$WORKDIR/$TESTNAME"
 137 else
 138     STATEDIR="$BUILD_DIR/test/$TESTNAME"
 139 fi
 140
 141 STATEFILE="$STATEDIR/.testdir"
 142 IMAGESTATEDIR="$STATEDIR/.."
 143 TESTLOG="$STATEDIR/test.log"
 144
 145 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 146     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 147     exit 1
 148 fi
 149 TESTID="${BASH_REMATCH[1]:?}"
 150
 151 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 152     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 153     exit 1
 154 fi
 155
 156 BASICTOOLS=(
 157     awk
 158     base64
 159     basename
 160     bash
 161     capsh
 162     cat
 163     chgrp
 164     chmod
 165     chown
 166     chroot
 167     cmp
 168     cp
 169     cryptsetup
 170     cut
 171     date
 172     dd
 173     dhclient
 174     diff
 175     dirname
 176     dmsetup
 177     echo
 178     env
 179     false
 180     find
 181     findmnt
 182     flock
 183     getconf
 184     getent
 185     getfacl
 186     getfattr
 187     grep
 188     grep
 189     gunzip
 190     gzip
 191     head
 192     hostname
 193     id
 194     ionice
 195     ip
 196     jq
 197     killall
 198     ldd
 199     ln
 200     ln
 201     loadkeys
 202     login
 203     losetup
 204     ls
 205     lsattr
 206     lsblk
 207     lz4cat
 208     mkdir
 209     mkfifo
 210     mknod
 211     mktemp
 212     modprobe
 213     mount
 214     mountpoint
 215     mv
 216     nc
 217     nproc
 218     ping
 219     pkill
 220     ps
 221     readlink
 222     realpath
 223     rev
 224     rm
 225     rmdir
 226     rmmod
 227     script
 228     sed
 229     seq
 230     setfacl
 231     setfattr
 232     setfont
 233     setpriv
 234     setsid
 235     sfdisk
 236     sh
 237     sleep
 238     sort
 239     stat
 240     stty
 241     su
 242     sulogin
 243     sysctl
 244     tail
 245     tar
 246     tee
 247     test
 248     timeout
 249     touch
 250     tr
 251     true
 252     truncate
 253     tty
 254     umount
 255     uname
 256     unshare
 257     useradd
 258     userdel
 259     wc
 260     whoami
 261     xargs
 262     xzcat
 263 )
 264
 265 DEBUGTOOLS=(
 266     df
 267     dmesg
 268     du
 269     free
 270     less
 271     strace
 272     vi
 273     /usr/libexec/vi
 274 )
 275
 276 is_built_with_asan() {
 277     local _bin="${1:?}"
 278
 279     if ! type -P objdump >/dev/null; then
 280         echo "Failed to find objdump, assuming systemd hasn't been built with ASAN."
 281         return 1
 282     fi
 283
 284     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 285     local _asan_calls
 286     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 287     if ((_asan_calls < 1000)); then
 288         return 1
 289     else
 290         return 0
 291     fi
 292 }
 293
 294 is_built_with_coverage() {
 295     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 296         return 1
 297     fi
 298
 299     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 300 }
 301
 302 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 303 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 304
 305 if get_bool "$IS_BUILT_WITH_ASAN"; then
 306     PATH_TO_INIT="$ROOTLIBDIR/systemd-under-asan"
 307     QEMU_MEM="${QEMU_MEM:-2G}"
 308     QEMU_SMP="${QEMU_SMP:-4}"
 309
 310     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 311     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 312         ASAN_COMPILER=gcc
 313         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 314     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 315         ASAN_COMPILER=clang
 316         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 317
 318         # As clang's ASan DSO is usually in a non-standard path, let's check if
 319         # the environment is set accordingly. If not, warn the user and exit.
 320         # We're not setting the LD_LIBRARY_PATH automagically here, because
 321         # user should encounter (and fix) the same issue when running the unit
 322         # tests (meson test)
 323         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 324             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 325             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 326             exit 1
 327         fi
 328     else
 329         echo >&2 "systemd is not linked against the ASan DSO"
 330         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 331         exit 1
 332     fi
 333
 334     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 335 fi
 336
 337 test_require_bin() {
 338     local bin
 339
 340     for bin in "$@"; do
 341         if ! command -v "$bin" >/dev/null; then
 342             echo "Required binary $bin not available, skipping the test"
 343             exit 0
 344         fi
 345     done
 346 }
 347
 348 find_qemu_bin() {
 349     QEMU_BIN="${QEMU_BIN:-""}"
 350     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 351     if get_bool "$QEMU_KVM"; then
 352         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 353     fi
 354
 355     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 356     case $ARCH in
 357     x86_64)
 358         # QEMU's own build system calls it qemu-system-x86_64
 359         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 360         ;;
 361     i*86)
 362         # new i386 version of QEMU
 363         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 364
 365         # i386 version of QEMU
 366         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 367         ;;
 368     ppc64*)
 369         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 370         ;;
 371     esac
 372
 373     if [[ ! -e "$QEMU_BIN" ]]; then
 374         echo "Could not find a suitable qemu binary" >&2
 375         return 1
 376     fi
 377 }
 378
 379 qemu_setup_swtpm_socket() {
 380     local pid state_dir tpm_device
 381
 382     if ! tpm_device="$(qemu_get_tpm_device)"; then
 383         dinfo "Found QEMU version is too old for TPM2 on ppc64le"
 384         exit 0
 385     fi
 386
 387     state_dir="$(mktemp -d)"
 388     swtpm socket --tpm2 --tpmstate dir="$state_dir" --ctrl type=unixio,path="$state_dir/sock" &
 389     pid=$!
 390     if ! kill -0 "$pid"; then
 391         derror "Failed to start swtpm"
 392         return 1
 393     fi
 394
 395     if ! timeout 5 bash -c "until [[ -S $state_dir/sock ]]; do sleep .5; done"; then
 396         derror "Failed to setup swtpm socket"
 397         return 1
 398     fi
 399
 400     dinfo "Started swtpm as PID $pid with state dir $state_dir"
 401
 402     add_at_exit_handler "kill -TERM $pid 2>/dev/null; rm -rf '$state_dir'"
 403
 404     QEMU_OPTIONS+=" -chardev socket,id=chrtpm,path=$state_dir/sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device $tpm_device,tpmdev=tpm0"
 405     dinfo "Configured emulated TPM2 device $tpm_device"
 406
 407     return 0
 408 }
 409
 410 qemu_get_tpm_device() {
 411     local tpm_device="tpm-tis"
 412
 413     if [[ "$(uname -m)" == "ppc64le" ]]; then
 414         # tpm-spapr support was introduced in qemu 5.0.0
 415         if ! qemu_min_version "5.0.0"; then
 416             return 1
 417         fi
 418
 419         tpm_device="tpm-spapr"
 420     fi
 421
 422     echo "$tpm_device"
 423     return 0
 424 }
 425
 426 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 427 # returns 0 if newer or equal
 428 # returns 1 if older
 429 # returns 2 if failing
 430 qemu_min_version() {
 431     find_qemu_bin || return 2
 432
 433     # get version from binary
 434     local qemu_ver
 435     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 436
 437     # Check version string format
 438     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 439     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 440
 441     # compare as last command to return that value
 442     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 443 }
 444
 445 # Pads a file to multiple of 4 bytes
 446 pad4_file() {
 447     local size
 448     size=$(stat -c "%s" "$1")
 449     local padded
 450     padded=$((((size + 3) / 4) * 4))
 451     truncate -s "$padded" "$1"
 452 }
 453
 454 # Return 0 if qemu did run (then you must check the result state/logs for actual
 455 # success), or 1 if qemu is not available.
 456 run_qemu() {
 457     if declare -F run_qemu_hook >/dev/null; then
 458         if ! run_qemu_hook "${workspace}"; then
 459             derror "check_qemu_hook() returned with EC > 0"
 460             ret=4
 461         fi
 462     fi
 463
 464     # If the test provided its own initrd, use it (e.g. TEST-24)
 465     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 466         INITRD="$TESTDIR/initrd.img"
 467     fi
 468
 469     if [ -f /etc/machine-id ]; then
 470         read -r MACHINE_ID </etc/machine-id
 471         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 472             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 473         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 474             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 475     fi
 476
 477     local CONSOLE=ttyS0
 478
 479     # Reset the boot counter, if present
 480     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 481     rm -f "$initdir"/{testok,failed,skipped}
 482     # make sure the initdir is not mounted to avoid concurrent access
 483     cleanup_initdir
 484     umount_loopback
 485
 486     if [[ ! "$KERNEL_BIN" ]]; then
 487         if get_bool "$LOOKS_LIKE_ARCH"; then
 488             KERNEL_BIN=/boot/vmlinuz-linux
 489         else
 490             [ "$ARCH" ] || ARCH=$(uname -m)
 491             case $ARCH in
 492                 ppc64*)
 493                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 494                     # uses the "standard" vmlinuz- prefix
 495                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 496                     CONSOLE=hvc0
 497                     ;;
 498                 *)
 499                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 500                     ;;
 501             esac
 502         fi
 503     fi
 504
 505     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 506     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 507     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 508     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 509     if [[ ! "$INITRD" ]]; then
 510         if [[ -e "$default_fedora_initrd" ]]; then
 511             INITRD="$default_fedora_initrd"
 512         elif get_bool "$LOOKS_LIKE_DEBIAN" && [[ -e "$default_debian_initrd" ]]; then
 513             INITRD="$default_debian_initrd"
 514         elif get_bool "$LOOKS_LIKE_ARCH" && [[ -e "$default_arch_initrd" ]]; then
 515             INITRD="$default_arch_initrd"
 516         elif get_bool "$LOOKS_LIKE_SUSE" && [[ -e "$default_suse_initrd" ]]; then
 517             INITRD="$default_suse_initrd"
 518         fi
 519     fi
 520
 521     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 522     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 523     # is not installed or there's some other error when calling it, fall back
 524     # to the original value (QEMU_SMP=1).
 525     if [[ -z "${QEMU_SMP:=}" ]]; then
 526         if ! QEMU_SMP=$(nproc); then
 527             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 528             QEMU_SMP=1
 529         fi
 530     fi
 531
 532     find_qemu_bin || return 1
 533
 534     if get_bool "${TEST_SETUP_SWTPM:-}"; then
 535         qemu_setup_swtpm_socket || return 1
 536     fi
 537
 538     # Umount initdir to avoid concurrent access to the filesystem
 539     _umount_dir "$initdir"
 540
 541     local kernel_params=()
 542     local qemu_options=()
 543     local qemu_cmd=("$QEMU_BIN")
 544
 545     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 546         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 547     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 548         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 549     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 550         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 551     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 552         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 553         exit 1
 554     fi
 555
 556     if get_bool "$LOOKS_LIKE_SUSE"; then
 557         kernel_params+=("rd.hostonly=0")
 558     fi
 559
 560     # Debian/Ubuntu's initramfs tries to check if it can resume from hibernation
 561     # and wastes a minute or so probing disks, skip that as it's not useful here
 562     kernel_params+=(
 563         "root=LABEL=systemd_boot"
 564         "rw"
 565         "raid=noautodetect"
 566         "rd.luks=0"
 567         "loglevel=2"
 568         "init=$PATH_TO_INIT"
 569         "console=$CONSOLE"
 570         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 571         "systemd.unit=testsuite.target"
 572         "systemd.wants=testsuite-$1.service"
 573         "noresume"
 574         "oops=panic"
 575         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 576         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 577     )
 578
 579     if ! get_bool "$INTERACTIVE_DEBUG" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 580         kernel_params+=(
 581             "panic=1"
 582             "softlockup_panic=1"
 583             "systemd.wants=end.service"
 584         )
 585     fi
 586
 587     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 588     qemu_options+=(
 589         -smp "$QEMU_SMP"
 590         -net none
 591         -m "${QEMU_MEM:-768M}"
 592         -nographic
 593         -kernel "$KERNEL_BIN"
 594         -drive "format=raw,cache=unsafe,file=$image"
 595         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 596     )
 597
 598     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 599         local user_qemu_options
 600         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 601         qemu_options+=("${user_qemu_options[@]}")
 602     fi
 603     qemu_options+=(${QEMU_OPTIONS_ARRAY:+"${QEMU_OPTIONS_ARRAY[@]}"})
 604
 605     if [[ -n "$INITRD" ]]; then
 606         if [[ -n "$INITRD_EXTRA" ]]; then
 607             # An addition initrd has been specified, let's combine it with the main one.
 608             local t="$WORKDIR"/initrd.combined."$RANDOM"
 609
 610             # First, show contents of additional initrd
 611             echo "Additional initrd contents:"
 612             cpio -tv < "$INITRD_EXTRA"
 613
 614             # Copy the main initrd
 615             zstd -d -c -f "$INITRD" > "$t"
 616             add_at_exit_handler "rm $t"
 617             # Kernel requires this to be padded to multiple of 4 bytes with zeroes
 618             pad4_file "$t"
 619
 620             # Copy the additional initrd
 621             cat "$INITRD_EXTRA" >> "$t"
 622             pad4_file "$t"
 623
 624             qemu_options+=(-initrd "$t")
 625         else
 626             qemu_options+=(-initrd "$INITRD")
 627         fi
 628     fi
 629
 630     # Let's use KVM if possible
 631     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 632         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 633     fi
 634
 635     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 636         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 637     fi
 638
 639     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]} ${KERNEL_APPEND:-}" |& tee "${TESTDIR:?}/console.log")
 640     rc=$?
 641     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 642         derror "Test timed out after ${QEMU_TIMEOUT}s"
 643         TIMED_OUT=1
 644     else
 645         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 646     fi
 647     return 0
 648 }
 649
 650 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 651 # success), or 1 if nspawn is not available.
 652 run_nspawn() {
 653     [[ -d /run/systemd/system ]] || return 1
 654     # Reset the boot counter, if present
 655     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 656     rm -f "${initdir:?}"/{testok,failed,skipped}
 657
 658     local nspawn_cmd=()
 659     local nspawn_options=(
 660         "--register=no"
 661         "--kill-signal=SIGKILL"
 662         "--directory=${1:?}"
 663         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 664         "--machine=TEST-$TESTID"
 665     )
 666     local kernel_params=(
 667         "$PATH_TO_INIT"
 668         "systemd.unit=testsuite.target"
 669         "systemd.wants=testsuite-$2.service"
 670         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 671         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 672     )
 673
 674     if get_bool "$INTERACTIVE_DEBUG"; then
 675         nspawn_options+=("--console=interactive")
 676     elif ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 677         kernel_params+=("systemd.wants=end.service")
 678     fi
 679
 680     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 681         local user_nspawn_arguments
 682         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 683         nspawn_options+=("${user_nspawn_arguments[@]}")
 684     fi
 685
 686     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 687         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 688         exit
 689     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 690         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 691     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 692         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 693     else
 694         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 695         exit 1
 696     fi
 697
 698     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 699         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 700     else
 701         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 702     fi
 703
 704     # Word splitting here is intentional
 705     # shellcheck disable=SC2086
 706     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}" ${KERNEL_APPEND:-} |& tee "${TESTDIR:?}/console.log")
 707     rc=$?
 708     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 709         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 710         TIMED_OUT=1
 711     else
 712         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 713     fi
 714     return 0
 715 }
 716
 717 # Build two very minimal root images, with two units, one is the same and one is different across them
 718 install_verity_minimal() {
 719     dinfo "Set up a set of minimal images for verity verification"
 720     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 721         return
 722     fi
 723     if ! command -v mksquashfs >/dev/null 2>&1; then
 724         dfatal "mksquashfs not found"
 725         exit 1
 726     fi
 727     if ! command -v veritysetup >/dev/null 2>&1; then
 728         dfatal "veritysetup not found"
 729         exit 1
 730     fi
 731     # Local modifications of some global variables is intentional in this
 732     # subshell (SC2030)
 733     # shellcheck disable=SC2030
 734     (
 735         BASICTOOLS=(
 736             bash
 737             cat
 738             echo
 739             grep
 740             mount
 741             sleep
 742             touch
 743         )
 744         oldinitdir="$initdir"
 745         rm -rfv "$TESTDIR/minimal"
 746         export initdir="$TESTDIR/minimal"
 747         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 748         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 749         setup_basic_dirs
 750         install_basic_tools
 751         install_ld_so_conf
 752         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 753         # issue, thus falsely triggering SC2030 in this case
 754         # See: koalaman/shellcheck#1409
 755         if [[ -v ASAN_RT_PATH ]]; then
 756             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 757             # into the verity images to get rid of the annoying errors about
 758             # missing $LD_PRELOAD libraries.
 759             inst_libs "$ASAN_RT_PATH"
 760             inst_library "$ASAN_RT_PATH"
 761         fi
 762         cp "$os_release" "$initdir/usr/lib/os-release"
 763         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 764         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 765         touch "$initdir/opt/some_file"
 766         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 767         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 768         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 769 [Service]
 770 ExecStartPre=cat /usr/lib/os-release
 771 ExecStart=sleep 120
 772 EOF
 773         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 774
 775         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 776         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 777             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 778
 779         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 780         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 781         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 782
 783         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 784         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 785             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 786
 787         # Rolling distros like Arch do not set VERSION_ID
 788         local version_id=""
 789         if grep -q "^VERSION_ID=" "$os_release"; then
 790             version_id="$(grep "^VERSION_ID=" "$os_release")"
 791         fi
 792
 793         export initdir="$TESTDIR/app0"
 794         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 795         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 796         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 797         ( echo "${version_id}"
 798           echo "SYSEXT_IMAGE_ID=app" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 799         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 800 [Service]
 801 Type=oneshot
 802 RemainAfterExit=yes
 803 ExecStart=/opt/script0.sh
 804 TemporaryFileSystem=/var/lib
 805 StateDirectory=app0
 806 RuntimeDirectory=app0
 807 EOF
 808         cat >"$initdir/opt/script0.sh" <<EOF
 809 #!/bin/bash
 810 set -e
 811 test -e /usr/lib/os-release
 812 echo bar >\${STATE_DIRECTORY}/foo
 813 cat /usr/lib/extension-release.d/extension-release.app0
 814 EOF
 815         chmod +x "$initdir/opt/script0.sh"
 816         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 817         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 818
 819         export initdir="$TESTDIR/conf0"
 820         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system" "$initdir/opt"
 821         grep "^ID=" "$os_release" >"$initdir/etc/extension-release.d/extension-release.conf0"
 822         echo "${version_id}" >>"$initdir/etc/extension-release.d/extension-release.conf0"
 823         ( echo "${version_id}"
 824           echo "CONFEXT_IMAGE_ID=app" ) >>"$initdir/etc/extension-release.d/extension-release.conf0"
 825         echo MARKER_1 >"$initdir/etc/systemd/system/some_file"
 826         mksquashfs "$initdir" "$oldinitdir/usr/share/conf0.raw" -noappend
 827
 828         export initdir="$TESTDIR/app1"
 829         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 830         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 831         ( echo "${version_id}"
 832           echo "SYSEXT_SCOPE=portable"
 833           echo "SYSEXT_IMAGE_ID=app"
 834           echo "SYSEXT_IMAGE_VERSION=1"
 835           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 836         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 837         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 838 [Service]
 839 Type=oneshot
 840 RemainAfterExit=yes
 841 ExecStart=/opt/script1.sh
 842 StateDirectory=app1
 843 RuntimeDirectory=app1
 844 EOF
 845         cat >"$initdir/opt/script1.sh" <<EOF
 846 #!/bin/bash
 847 set -e
 848 test -e /usr/lib/os-release
 849 echo baz >\${STATE_DIRECTORY}/foo
 850 cat /usr/lib/extension-release.d/extension-release.app2
 851 EOF
 852         chmod +x "$initdir/opt/script1.sh"
 853         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 854         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 855
 856         export initdir="$TESTDIR/app-nodistro"
 857         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 858         ( echo "ID=_any"
 859           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 860         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 861         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 862
 863         export initdir="$TESTDIR/service-scoped-test"
 864         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system"
 865         ( echo "ID=_any"
 866           echo "ARCHITECTURE=_any" ) >"$initdir/etc/extension-release.d/extension-release.service-scoped-test"
 867         echo MARKER_CONFEXT_123 >"$initdir/etc/systemd/system/some_file"
 868         mksquashfs "$initdir" "$oldinitdir/etc/service-scoped-test.raw" -noappend
 869
 870         # We need to create a dedicated sysext image to test the reload mechanism. If we share an image to install the
 871         # 'foo.service' it will be loaded from another test run, which will impact the targeted test.
 872         export initdir="$TESTDIR/app-reload"
 873         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 874         ( echo "ID=_any"
 875           echo "ARCHITECTURE=_any"
 876           echo "EXTENSION_RELOAD_MANAGER=1" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-reload"
 877         mkdir -p "$initdir/usr/lib/systemd/system/multi-user.target.d"
 878         cat >"${initdir}/usr/lib/systemd/system/foo.service" <<EOF
 879 [Service]
 880 Type=oneshot
 881 RemainAfterExit=yes
 882 ExecStart=echo foo
 883
 884 [Install]
 885 WantedBy=multi-user.target
 886 EOF
 887         { echo "[Unit]"; echo "Upholds=foo.service"; } > "$initdir/usr/lib/systemd/system/multi-user.target.d/10-foo-service.conf"
 888         mksquashfs "$initdir" "$oldinitdir/usr/share/app-reload.raw" -noappend
 889     )
 890 }
 891
 892 setup_basic_environment() {
 893     # create the basic filesystem layout
 894     setup_basic_dirs
 895
 896     install_systemd
 897     install_missing_libraries
 898     install_config_files
 899     install_zoneinfo
 900     create_rc_local
 901     install_basic_tools
 902     install_libnss
 903     install_pam
 904     install_dbus
 905     install_fonts
 906     install_locales
 907     install_keymaps
 908     install_x11_keymaps
 909     install_terminfo
 910     install_execs
 911     install_fs_tools
 912     install_modules
 913     install_plymouth
 914     install_haveged
 915     install_debug_tools
 916     install_ld_so_conf
 917     install_testuser
 918     has_user_dbus_socket && install_user_dbus
 919     setup_selinux
 920     install_depmod_files
 921     generate_module_dependencies
 922     if get_bool "$IS_BUILT_WITH_ASAN"; then
 923         create_asan_wrapper
 924     fi
 925     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 926         install_verity_minimal
 927     fi
 928 }
 929
 930 setup_selinux() {
 931     dinfo "Setup SELinux"
 932     # don't forget KERNEL_APPEND='... selinux=1 ...'
 933     if ! get_bool "$SETUP_SELINUX"; then
 934         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 935         return 0
 936     fi
 937
 938     for dir in /etc/selinux /usr/share/selinux; do
 939         rm -rf "${initdir:?}/$dir"
 940         if ! cp -ar "$dir" "$initdir/$dir"; then
 941             dfatal "Failed to copy $dir"
 942             exit 1
 943         fi
 944     done
 945
 946     # We use a custom autorelabel service instead of the SELinux provided set
 947     # of units & a generator, since the generator overrides the default target
 948     # to the SELinux one when it detects /.autorelabel. However, we use
 949     # systemd.unit= on the kernel command cmdline which always takes precedence,
 950     # rendering all SELinux efforts useless. Also, pulling in selinux-autorelabel.service
 951     # explicitly doesn't work either, as it doesn't check for the presence of /.autorelabel
 952     # and does the relabeling unconditionally which always ends with a reboot, so
 953     # we end up in a reboot loop (and it also spews quite a lot of errors as it
 954     # wants /etc/fstab and dracut-initramfs-restore).
 955     touch "$initdir/.autorelabel"
 956     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 957     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 958
 959     # Tools requires by fixfiles
 960     image_install awk bash cat chcon expr egrep find grep head secon setfiles rm sort uname uniq
 961     image_install fixfiles getenforce load_policy selinuxenabled sestatus
 962 }
 963
 964 install_valgrind() {
 965     if ! type -p valgrind; then
 966         dfatal "Failed to install valgrind"
 967         exit 1
 968     fi
 969
 970     local valgrind_bins valgrind_libs valgrind_supp
 971
 972     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null |
 973                                        sed -r -n 's/execve\("([^"]*)".*/\1/p')
 974     image_install "${valgrind_bins[@]}"
 975
 976     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null |
 977                                        sed -r -n 's|.*calling init: (/.*vgpreload_.*)|\1|p')
 978     image_install "${valgrind_libs[@]}"
 979
 980     readarray -t valgrind_supp < <(strace -e open valgrind /bin/true 2>&1 >/dev/null |
 981                                        sed -r -n 's,open\("([^"]*(/debug[^"]*|\.supp))".*= [0-9].*,\1,p')
 982     image_install "${valgrind_supp[@]}"
 983 }
 984
 985 create_valgrind_wrapper() {
 986     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 987     ddebug "Create $valgrind_wrapper"
 988     cat >"$valgrind_wrapper" <<EOF
 989 #!/usr/bin/env bash
 990
 991 mount -t proc proc /proc
 992 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 993 EOF
 994     chmod 0755 "$valgrind_wrapper"
 995 }
 996
 997 create_asan_wrapper() {
 998     local asan_wrapper default_asan_options default_ubsan_options default_environment manager_environment
 999
1000     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
1001
1002     default_asan_options="${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1}"
1003     default_ubsan_options="${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}"
1004
1005     if [[ "$ASAN_COMPILER" == "clang" ]]; then
1006         # clang: install llvm-symbolizer to generate useful reports
1007         # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
1008         image_install "llvm-symbolizer"
1009
1010         # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
1011         # unnecessary for gcc & libasan, however, for clang this is crucial, as its
1012         # runtime ASan DSO is in a non-standard (library) path.
1013         mkdir -p "${initdir:?}/etc/ld.so.conf.d/"
1014         echo "${ASAN_RT_PATH%/*}" >"${initdir:?}/etc/ld.so.conf.d/asan-path-override.conf"
1015         ldconfig -r "$initdir"
1016     fi
1017
1018     # Create a simple environment file which can be included by systemd services
1019     # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
1020     cat >"${initdir:?}/usr/lib/systemd/systemd-asan-env" <<EOF
1021 LD_PRELOAD=$ASAN_RT_PATH
1022 ASAN_OPTIONS=$default_asan_options
1023 LSAN_OPTIONS=detect_leaks=0
1024 UBSAN_OPTIONS=$default_ubsan_options
1025 EOF
1026
1027     default_environment=(
1028         "ASAN_OPTIONS='$default_asan_options'"
1029         "UBSAN_OPTIONS='$default_ubsan_options'"
1030         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1031     )
1032     manager_environment=(
1033         "ASAN_OPTIONS='$default_asan_options:log_path=/systemd-pid1.asan.log:log_to_syslog=1'"
1034         "UBSAN_OPTIONS='$default_ubsan_options:log_path=/systemd-pid1.ubsan.log:log_to_syslog=1'"
1035         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1036     )
1037
1038     mkdir -p "${initdir:?}/etc/systemd/system.conf.d/"
1039     cat >"${initdir:?}/etc/systemd/system.conf.d/asan.conf" <<EOF
1040 [Manager]
1041 DefaultEnvironment=${default_environment[*]}
1042 ManagerEnvironment=${manager_environment[*]}
1043 DefaultTimeoutStartSec=180s
1044 EOF
1045
1046     # ASAN and syscall filters aren't compatible with each other.
1047     find "${initdir:?}" -name '*.service' -type f -print0 | xargs -0 sed -i 's/^\(MemoryDeny\|SystemCall\)/#\1/'
1048
1049     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journald.service.d/"
1050     cat >"${initdir:?}/etc/systemd/system/systemd-journald.service.d/asan-env.conf" <<EOF
1051 [Service]
1052 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
1053 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
1054 Environment=ASAN_OPTIONS=$default_asan_options:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=$default_ubsan_options:log_path=/systemd-journald.ubsan.log
1055
1056 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
1057 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
1058 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
1059 StandardOutput=file:/systemd-journald.out
1060 EOF
1061
1062     # 90s isn't enough for some services to finish when literally everything is run
1063     # under ASan+UBSan in containers, which, in turn, are run in VMs.
1064     # Let's limit which environments such services should be executed in.
1065     mkdir -p "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/"
1066     cat >"${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/asan.conf" <<EOF
1067 [Unit]
1068 ConditionVirtualization=container
1069
1070 [Service]
1071 TimeoutSec=240s
1072 EOF
1073
1074     # Let's override another hard-coded timeout that kicks in too early
1075     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/"
1076     cat >"${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/asan.conf" <<EOF
1077 [Service]
1078 TimeoutSec=180s
1079 EOF
1080
1081     asan_wrapper="${initdir:?}/${PATH_TO_INIT:?}"
1082     # Sanity check to make sure we don't overwrite something we shouldn't.
1083     [[ "$asan_wrapper" =~ systemd-under-asan$ ]]
1084
1085     cat >"$asan_wrapper" <<EOF
1086 #!/usr/bin/env bash
1087 set -eux
1088
1089 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
1090 export ${manager_environment[@]}
1091 [[ -n "\$ASAN_OPTIONS" && -n "\$UBSAN_OPTIONS" ]]
1092
1093 exec "$ROOTLIBDIR/systemd" "\$@"
1094 EOF
1095     chmod 0755 "$asan_wrapper"
1096 }
1097
1098 create_strace_wrapper() {
1099     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
1100     ddebug "Create $strace_wrapper"
1101     cat >"$strace_wrapper" <<EOF
1102 #!/usr/bin/env bash
1103
1104 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
1105 EOF
1106     chmod 0755 "$strace_wrapper"
1107 }
1108
1109 install_fs_tools() {
1110     dinfo "Install fsck"
1111     image_install /sbin/fsck*
1112     image_install -o /bin/fsck*
1113
1114     # fskc.reiserfs calls reiserfsck. so, install it
1115     image_install -o reiserfsck
1116
1117     # we use mkfs in system-repart tests
1118     image_install /sbin/mkfs.ext4
1119     image_install /sbin/mkfs.vfat
1120 }
1121
1122 install_modules() {
1123     dinfo "Install modules"
1124
1125     instmods bridge dummy ext4 ipvlan macvlan vfat veth
1126     instmods loop =block
1127     instmods nls_ascii =nls
1128     instmods overlay =overlayfs
1129     instmods scsi_debug
1130
1131     if get_bool "$LOOKS_LIKE_SUSE"; then
1132         instmods af_packet
1133     fi
1134 }
1135
1136 install_dmevent() {
1137     instmods dm_crypt =crypto
1138     inst_binary dmeventd
1139     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
1140     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1141         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
1142         # and since buster/bionic 95-dm-notify.rules
1143         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
1144         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
1145     else
1146         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
1147     fi
1148     if get_bool "$LOOKS_LIKE_SUSE"; then
1149         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
1150     fi
1151 }
1152
1153 install_multipath() {
1154     instmods "=md" multipath
1155     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
1156     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
1157     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1158         # Note: try both 60-kpartx.rules (as seen on Debian Sid with 0.9.4-7) and 95-kpartx.rules (as seen on
1159         # Ubuntu Jammy with 0.8.8-1ubuntu1.22.04.4)
1160         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-kpartx.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
1161     else
1162         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
1163     fi
1164     mkdir -p "${initdir:?}/etc/multipath"
1165
1166     local file
1167     while read -r file; do
1168         # Install libraries required by the given library
1169         inst_libs "$file"
1170         # Install the library itself and create necessary symlinks
1171         inst_library "$file"
1172     done < <(find /lib*/multipath -type f)
1173 }
1174
1175 install_lvm() {
1176     local lvm_rules rule_prefix
1177
1178     image_install lvm
1179     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1180     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1181     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1182
1183     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1184         lvm_rules="56-lvm.rules"
1185         rule_prefix=""
1186     else
1187         lvm_rules="11-dm-lvm.rules"
1188         rule_prefix="dm-"
1189     fi
1190
1191     # Support the new udev autoactivation introduced in lvm 2.03.14
1192     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1193     # Static autoactivation (via lvm2-activation-generator) was dropped
1194     # in lvm 2.03.15
1195     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1196     if [[ -f "/lib/udev/rules.d/69-${rule_prefix}lvm.rules" ]]; then
1197         inst_rules "$lvm_rules" "69-${rule_prefix}lvm.rules"
1198     else
1199         image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1200         image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1201         inst_rules "$lvm_rules" "69-${rule_prefix}lvm-metad.rules"
1202     fi
1203
1204     mkdir -p "${initdir:?}/etc/lvm"
1205 }
1206
1207 host_has_btrfs() (
1208     set -e
1209     modprobe -nv btrfs && command -v mkfs.btrfs && command -v btrfs || return $?
1210 )
1211
1212 install_btrfs() {
1213     instmods btrfs
1214     # Not all utilities provided by btrfs-progs are listed here; extend the list
1215     # if necessary
1216     image_install btrfs btrfstune mkfs.btrfs
1217     inst_rules 64-btrfs-dm.rules
1218 }
1219
1220 install_iscsi() {
1221     # Install both client and server side stuff by default
1222     local inst="${1:-}"
1223     local file
1224
1225     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1226     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1227     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1228         image_install iscsi-iname iscsiadm iscsid iscsistart
1229         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1230         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1231         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1232         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1233         mkdir -p "${initdir:?}/etc/iscsi"
1234         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1235         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1236         # a one-time service instead of distro package's post-install scripts.
1237         # However, some distros still use this approach even after this patch,
1238         # so prefer the already existing initiatorname.iscsi file if it exists.
1239         #
1240         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1241         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1242             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1243             if get_bool "$IS_BUILT_WITH_ASAN"; then
1244                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1245                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1246                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1247                 # (among others) pre-loads ASan's DSO.
1248                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1249                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1250             fi
1251         else
1252             inst_simple "/etc/iscsi/initiatorname.iscsi"
1253         fi
1254     fi
1255
1256     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1257     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1258     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1259         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1260         image_install -o /etc/sysconfig/tgtd
1261         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1262         mkdir -p "${initdir:?}/etc/tgt"
1263         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1264         # Install perl modules required by tgt-admin
1265         #
1266         # Forgive me father for I have sinned. The monstrosity below appends
1267         # a perl snippet to the `tgt-admin` perl script on the fly, which
1268         # dumps a list of files (perl modules) required by `tgt-admin` at
1269         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1270         # passed to `inst_simple` which installs the necessary files into the image
1271         #
1272         # shellcheck disable=SC2016
1273         while read -r file; do
1274             inst_simple "$file"
1275         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1276     fi
1277 }
1278
1279 host_has_mdadm() (
1280     set -e
1281     command -v mdadm || return $?
1282 )
1283
1284 install_mdadm() {
1285     local unit
1286     local mdadm_units=(
1287         system/mdadm-grow-continue@.service
1288         system/mdadm-last-resort@.service
1289         system/mdadm-last-resort@.timer
1290         system/mdmon@.service
1291         system/mdmonitor-oneshot.service
1292         system/mdmonitor-oneshot.timer
1293         system/mdmonitor.service
1294         system-shutdown/mdadm.shutdown
1295     )
1296
1297     instmods "=md"
1298     image_install mdadm mdmon
1299     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1300     # Fedora/CentOS/RHEL ships this rule file
1301     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1302
1303     for unit in "${mdadm_units[@]}"; do
1304         image_install "${ROOTLIBDIR:?}/$unit"
1305     done
1306
1307     # Disable the mdmonitor service, since it fails if there's no valid email address
1308     # configured in /etc/mdadm.conf, which just unnecessarily pollutes the logs
1309     "${SYSTEMCTL:?}" mask --root "${initdir:?}" mdmonitor.service || :
1310 }
1311
1312 install_compiled_systemd() {
1313     dinfo "Install compiled systemd"
1314
1315     local ninja_bin
1316     ninja_bin="$(type -P ninja || type -P ninja-build)"
1317     if [[ -z "$ninja_bin" ]]; then
1318         dfatal "ninja was not found"
1319         exit 1
1320     fi
1321     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1322
1323     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1324     # find them in the same directory as the runtime data counts
1325     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1326         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1327         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1328         # Set effective & default ACLs for the build dir so unprivileged
1329         # processes can write gcda files with coverage stats
1330         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1331     fi
1332 }
1333
1334 install_package_file() {
1335     local file="${1:?}"
1336
1337     # Skip missing files (like /etc/machine-info)
1338     [[ ! -e "$file" ]] && return 0
1339     # Skip python unit tests, since the image_install machinery will try to pull
1340     # in the whole python stack in a very questionable state, making the tests fail.
1341     # And given we're trying to transition to mkosi-based images anyway I'm not even
1342     # going to bother
1343     [[ "$file" =~ /tests/unit-tests/.*.py$ ]] && return 0
1344     # If the current file is a directory, create it with the original
1345     # mode; if it's a symlink to a directory, copy it as-is
1346     if [[ -d "$file" ]]; then
1347         inst_dir "$file"
1348     else
1349         inst "$file"
1350     fi
1351 }
1352
1353 install_debian_systemd() {
1354     dinfo "Install debian systemd"
1355
1356     local deb file
1357
1358     while read -r deb; do
1359         ddebug "Install debian files from package $deb"
1360         while read -r file; do
1361             install_package_file "$file"
1362         done < <(dpkg-query -L "$deb" 2>/dev/null)
1363     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1364 }
1365
1366 install_rpm() {
1367     local rpm="${1:?}"
1368     local file
1369
1370     if ! rpm -q "$rpm" >/dev/null; then
1371         derror "RPM $rpm is not installed"
1372         return 1
1373     fi
1374
1375     dinfo "Installing contents of RPM $rpm"
1376     while read -r file; do
1377         install_package_file "$file"
1378     done < <(rpm -ql "$rpm")
1379 }
1380
1381 install_suse_systemd() {
1382     local pkgs
1383
1384     dinfo "Install basic filesystem structure"
1385     install_rpm filesystem
1386
1387     dinfo "Install SUSE systemd"
1388
1389     pkgs=(
1390         systemd
1391         systemd-boot
1392         systemd-container
1393         systemd-coredump
1394         systemd-experimental
1395         systemd-homed
1396         systemd-journal-remote
1397         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1398         # systemd-network sub-package on openSUSE, has its own testsuite.
1399         systemd-network
1400         systemd-portable
1401         udev
1402     )
1403
1404     for p in "${pkgs[@]}"; do
1405         rpm -q "$p" &>/dev/null || continue
1406
1407         install_rpm "$p"
1408     done
1409
1410     dinfo "Install the data needed by the tests at runtime"
1411     inst_recursive "${SOURCE_DIR}/testdata"
1412     inst_recursive "${SOURCE_DIR}/unit-tests/manual"
1413
1414     # On openSUSE, this directory is not created at package install, at least
1415     # for now.
1416     mkdir -p "$initdir/var/log/journal/remote"
1417 }
1418
1419 install_fedora_systemd() {
1420     local required_packages=(
1421         systemd
1422         systemd-container
1423         systemd-libs
1424         systemd-pam
1425         systemd-tests
1426         systemd-udev
1427     )
1428     local optional_packages=(
1429         systemd-boot-unsigned
1430         systemd-bootchart
1431         systemd-journal-remote
1432         systemd-networkd
1433         systemd-oomd-defaults
1434         systemd-resolved
1435     )
1436     local package
1437
1438     for package in "${required_packages[@]}"; do
1439         install_rpm "$package"
1440     done
1441
1442     for package in "${optional_packages[@]}"; do
1443         rpm -q "$package" >/dev/null || continue
1444         install_rpm "$package"
1445     done
1446 }
1447
1448 install_distro_systemd() {
1449     dinfo "Install distro systemd"
1450
1451     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1452         install_debian_systemd
1453     elif get_bool "$LOOKS_LIKE_SUSE"; then
1454         install_suse_systemd
1455     elif get_bool "$LOOKS_LIKE_FEDORA"; then
1456         install_fedora_systemd
1457     else
1458         dfatal "NO_BUILD not supported for this distro"
1459         exit 1
1460     fi
1461 }
1462
1463 install_systemd() {
1464     dinfo "Install systemd"
1465     if get_bool "$NO_BUILD"; then
1466         install_distro_systemd
1467     else
1468         install_compiled_systemd
1469     fi
1470
1471     # Remove unneeded documentation
1472     rm -fr "${initdir:?}"/usr/share/{man,doc}
1473
1474     # Enable debug logging in PID1
1475     mkdir -p "$initdir/etc/systemd/system.conf.d/"
1476     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/system.conf.d/10-log-level.conf"
1477     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1478         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf.d/99-log-level.conf"
1479     fi
1480     # Enable debug logging for user instances as well
1481     mkdir -p "$initdir/etc/systemd/user.conf.d/"
1482     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/user.conf.d/10-log-level.conf"
1483     # Store coredumps in journal
1484     mkdir -p "$initdir/etc/systemd/coredump.conf.d/"
1485     echo -ne "[Coredump]\nStorage=journal\n" >"$initdir/etc/systemd/coredump.conf.d/10-storage-journal.conf"
1486     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1487     mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
1488     echo -ne "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/99-SYSTEMD_UNIT_PATH.conf"
1489
1490     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1491     # images, since it prevents gcov to write the coverage reports (*.gcda
1492     # files)
1493     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1494         mkdir -p "$initdir/etc/systemd/system/service.d/"
1495         echo -ne "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1496         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image to make the coverage work with
1497         # units using DynamicUser=yes. Do this only for services with test- prefix and a couple of
1498         # known-to-use DynamicUser=yes services, as setting this system-wide has many undesirable
1499         # side-effects, as it creates its own namespace.
1500         for service in capsule@ test- systemd-journal-{gatewayd,upload}; do
1501             mkdir -p "$initdir/etc/systemd/system/$service.service.d/"
1502             echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/$service.service.d/99-gcov-rwpaths-override.conf"
1503         done
1504         # Ditto, but for the user daemon
1505         mkdir -p "$initdir/etc/systemd/user/test-.service.d/"
1506         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf"
1507         # Bind the $BUILD_DIR into nspawn containers that are executed using
1508         # machinectl. Unfortunately, the .nspawn files don't support drop-ins
1509         # so we have to inject the bind mount directly into
1510         # the systemd-nspawn@.service unit.
1511         cp "$initdir/usr/lib/systemd/system/systemd-nspawn@.service" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1512         sed -ri "s/^ExecStart=.+$/& --bind=${BUILD_DIR//\//\\\/}/" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1513         # Pass the $BUILD_DIR as $COVERAGE_BUILD_DIR env variable to the system
1514         # manager, similarly to what we do with $ASAN_RT_PATH during sanitized
1515         # builds
1516         mkdir -p "$initdir/etc/systemd/system.conf.d/"
1517         echo -ne "[Manager]\nDefaultEnvironment=COVERAGE_BUILD_DIR=$BUILD_DIR\n" >"$initdir/etc/systemd/system.conf.d/99-COVERAGE_BUILD_DIR.conf"
1518     fi
1519
1520     # If we're built with -Dportabled=false, tests with systemd-analyze
1521     # --profile will fail. Since we need just the profile (text) files, let's
1522     # copy them into the image if they don't exist there.
1523     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1524     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1525         dinfo "Couldn't find portable profiles in the test image"
1526         dinfo "Copying them directly from the source tree"
1527         mkdir -p "$portable_dir"
1528         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1529     fi
1530 }
1531
1532 get_ldpath() {
1533     local rpath
1534     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1535
1536     if [ -z "$rpath" ] ; then
1537         echo "$BUILD_DIR"
1538     else
1539         echo "$rpath"
1540     fi
1541 }
1542
1543 install_missing_libraries() {
1544     dinfo "Install missing libraries"
1545     # install possible missing libraries
1546     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/unit-tests/{,manual/,unsafe/}}*; do
1547         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1548     done
1549
1550     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1551     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1552     # if not present
1553     # See: https://github.com/systemd/systemd/pull/23858
1554     while read -r libgcc_s; do
1555         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1556     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1557
1558     local lib path
1559     # A number of dependencies is now optional via dlopen, so the install
1560     # script will not pick them up, since it looks at linkage.
1561     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1 libarchive libgcrypt libkmod; do
1562         ddebug "Searching for $lib via pkg-config"
1563         if pkg-config --exists "$lib"; then
1564                 path="$(pkg-config --variable=libdir "$lib")"
1565                 if [ -z "${path}" ]; then
1566                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1567                     continue
1568                 fi
1569
1570                 if ! [[ ${lib} =~ ^lib ]]; then
1571                         lib="lib${lib}"
1572                 fi
1573                 # p11-kit-1's .so doesn't have the API level in the name
1574                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1575                         lib="libp11-kit"
1576                 fi
1577                 # Some pkg-config files are broken and give out the wrong paths
1578                 # (eg: libcryptsetup), so just ignore them
1579                 inst_libs "${path}/${lib}.so" || true
1580                 inst_library "${path}/${lib}.so" || true
1581
1582                 if [[ "$lib" == "libxkbcommon" ]]; then
1583                     install_x11_keymaps full
1584                 fi
1585         else
1586             ddebug "$lib.pc not found, skipping"
1587             continue
1588         fi
1589     done
1590
1591     # Install extra openssl 3 stuff
1592     path="$(pkg-config --variable=libdir libcrypto)"
1593     inst_simple "${path}/ossl-modules/legacy.so" || true
1594     inst_simple "${path}/ossl-modules/fips.so" || true
1595     inst_simple "${path}/engines-3/afalg.so" || true
1596     inst_simple "${path}/engines-3/capi.so" || true
1597     inst_simple "${path}/engines-3/loader_attic.so" || true
1598     inst_simple "${path}/engines-3/padlock.so" || true
1599
1600     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1601     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1602     # shellcheck disable=SC2046
1603     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1604 }
1605
1606 cleanup_loopdev() {
1607     if [ -n "${LOOPDEV:=}" ]; then
1608         ddebug "losetup -d $LOOPDEV"
1609         losetup -d "${LOOPDEV}"
1610         unset LOOPDEV
1611     fi
1612 }
1613
1614 add_at_exit_handler cleanup_loopdev
1615
1616 create_empty_image() {
1617     if [[ -z "${IMAGE_NAME:=}" ]]; then
1618         echo "create_empty_image: \$IMAGE_NAME not set"
1619         exit 1
1620     fi
1621
1622     # Partition sizes are in MiBs
1623     local root_size=768
1624     local data_size=100
1625     local esp_size=128
1626     local boot_size=128
1627     local total=
1628     if ! get_bool "$NO_BUILD"; then
1629         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1630             root_size=$((root_size + 200))
1631         fi
1632         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1633             root_size=$((root_size + 200))
1634         fi
1635         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1636             root_size=$((root_size + 250))
1637         fi
1638         if get_bool "$IS_BUILT_WITH_ASAN"; then
1639             root_size=$((root_size * 2))
1640         fi
1641     fi
1642
1643     if [[ "${IMAGE_ADDITIONAL_ROOT_SIZE:-0}" -gt 0 ]]; then
1644         root_size=$((root_size + IMAGE_ADDITIONAL_ROOT_SIZE))
1645     fi
1646     if [[ "${IMAGE_ADDITIONAL_DATA_SIZE:-0}" -gt 0 ]]; then
1647         data_size=$((data_size + IMAGE_ADDITIONAL_DATA_SIZE))
1648     fi
1649
1650     total=$((root_size + data_size + esp_size + boot_size))
1651
1652     echo "Setting up ${IMAGE_PUBLIC:?} (${total} MB)"
1653     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1654
1655     # Create the blank file to use as a root filesystem
1656     truncate -s "${total}M" "$IMAGE_PUBLIC"
1657
1658     LOOPDEV="$(losetup --show -P -f "$IMAGE_PUBLIC")"
1659     [[ -b "$LOOPDEV" ]] || return 1
1660     # Create two partitions - a root one and a data one (utilized by some tests)
1661     sfdisk "$LOOPDEV" <<EOF
1662 label: gpt
1663 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B name=esp  size=${esp_size}M
1664 type=$("${SYSTEMD_ID128:?}" show root -Pu) name=root size=${root_size}M bootable
1665 type=BC13C2FF-59E6-4262-A352-B275FD6F7172 name=boot size=${boot_size}M
1666 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1667 EOF
1668
1669     udevadm settle
1670
1671     if ! mkfs -t vfat "${LOOPDEV}p1"; then
1672         dfatal "Failed to mkfs -t vfat ${LOOPDEV}p1"
1673         exit 1
1674     fi
1675
1676     local label=(-L systemd_boot)
1677     # mkfs.reiserfs doesn't know -L. so, use --label instead
1678     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1679     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p2" -q; then
1680         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p2 -q"
1681         exit 1
1682     fi
1683
1684     local label=(-L xbootldr)
1685     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label xbootldr)
1686     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p3" -q; then
1687         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p3 -q"
1688         exit 1
1689     fi
1690 }
1691
1692 mount_initdir() {
1693     if [ -z "${LOOPDEV:=}" ]; then
1694         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1695         LOOPDEV="$(losetup --show -P -f "$image")"
1696         [ -b "$LOOPDEV" ] || return 1
1697
1698         udevadm settle
1699     fi
1700
1701     if ! mountpoint -q "${initdir:?}"; then
1702         mkdir -p "$initdir"
1703         mount "${LOOPDEV}p2" "$initdir"
1704         TEST_SETUP_CLEANUP_ROOTDIR=1
1705     fi
1706 }
1707
1708 cleanup_initdir() {
1709     # only umount if create_empty_image_rootdir() was called to mount it
1710     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1711         _umount_dir "${initdir:?}"
1712     fi
1713 }
1714
1715 umount_loopback() {
1716     # unmount the loopback device from all places. Otherwise we risk file
1717     # system corruption.
1718     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1719         ddebug "Unmounting all uses of $device"
1720         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1721     done
1722 }
1723
1724 create_empty_image_rootdir() {
1725     create_empty_image
1726     mount_initdir
1727 }
1728
1729 check_asan_reports() {
1730     local ret=0
1731     local root="${1:?}"
1732     local log report
1733
1734     if get_bool "$IS_BUILT_WITH_ASAN"; then
1735         ls -l "$root"
1736         if [[ -e "$root/systemd.asan.log.1" ]]; then
1737             cat "$root/systemd.asan.log.1"
1738             ret=$((ret+1))
1739         fi
1740
1741         for log in pid1 journald; do
1742             report="$(find "$root" -name "systemd-$log.*san.log*" -exec cat {} \;)"
1743             if [[ -n "$report" ]]; then
1744                 printf "%s\n" "$report"
1745                 # shellcheck disable=SC2015
1746                 [[ "$log" == journald ]] && cat "$root/systemd-journald.out" || :
1747                 ret=$((ret+1))
1748             fi
1749         done
1750
1751         # May 08 13:23:31 H testleak[2907148]: SUMMARY: AddressSanitizer: 4 byte(s) leaked in 1 allocation(s).
1752         pids="$(
1753             "$JOURNALCTL" -D "$root/var/log/journal" --grep 'SUMMARY: .*Sanitizer:' |
1754                    grep -v -E 'dbus-daemon|dbus-broker-launch' |
1755                    sed -r -n 's/.* .+\[([0-9]+)\]: SUMMARY:.*/\1/p'
1756         )"
1757
1758         if [[ -n "$pids" ]]; then
1759             ret=$((ret+1))
1760             for pid in $pids; do
1761                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1762             done
1763         fi
1764     fi
1765
1766     return $ret
1767 }
1768
1769 check_coverage_reports() {
1770     local root="${1:?}"
1771
1772     if get_bool "$NO_BUILD"; then
1773         return 0
1774     fi
1775     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1776         return 0
1777     fi
1778
1779     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1780         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1781     else
1782         dest="${TESTDIR:?}/coverage-info"
1783     fi
1784
1785     if [[ ! -e "${TESTDIR:?}/coverage-base" ]]; then
1786         # This shouldn't happen, as the report is generated during the setup
1787         # phase (test_setup()).
1788         derror "Missing base coverage report"
1789         return 1
1790     fi
1791
1792     # Create a coverage report that will later be uploaded. Remove info about system
1793     # libraries/headers and generated files, as we don't really care about them.
1794     lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1795     if [[ -f "$dest" ]]; then
1796         # If the destination report file already exists, don't overwrite it, but
1797         # merge it with the already present one - this usually happens when
1798         # running both "parts" of a test in one run (the qemu and the nspawn part).
1799         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1800     else
1801         # If there's no prior coverage report, merge the new one with the base
1802         # report we did during the setup phase (see test_setup()).
1803         lcov --add-tracefile "${TESTDIR:?}/coverage-base" --add-tracefile "${dest}.new" -o "${dest}"
1804     fi
1805     lcov --remove "$dest" -o "$dest" '/usr/include/*' '/usr/lib/*' "${BUILD_DIR:?}/*"
1806     rm -f "${dest}.new"
1807
1808     # If the test logs contain lines like:
1809     #
1810     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1811     #
1812     # it means we're possibly missing some coverage since gcov can't write the stats,
1813     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1814     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1815     # `setfacl` stuff in install_compiled_systemd().
1816     #
1817     # Also, a note: some tests, like TEST-46, overmount /home with tmpfs, which
1818     # means if your build dir is under /home/your-user (which is usually the
1819     # case) you might get bogus errors and missing coverage.
1820     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1821        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1822         derror "Detected possibly missing coverage, check the journal"
1823         return 1
1824     fi
1825
1826     return 0
1827 }
1828
1829 save_journal() {
1830     local source_dir="${1:?}"
1831     local state="${2:?}"
1832     # Default to always saving journal
1833     local save="yes"
1834     local dest_dir dest_name dest
1835
1836     if [[ "${TEST_SAVE_JOURNAL:-}" == "no" ]]; then
1837         save="no"
1838     elif [[ "${TEST_SAVE_JOURNAL:-}" == "fail" && "$state" -eq 0 ]]; then
1839         save="no"
1840     fi
1841
1842     if [[ -n "${ARTIFACT_DIRECTORY:-}" ]]; then
1843         dest_dir="$ARTIFACT_DIRECTORY"
1844         dest_name="${testname:?}.journal"
1845     else
1846         dest_dir="${TESTDIR:?}"
1847         dest_name="system.journal"
1848     fi
1849
1850     # Show messages from the testsuite-XX.service or messages with priority "warning" and higher
1851     echo " --- $source_dir ---"
1852     "$JOURNALCTL" --all --no-pager --no-hostname -o short-monotonic -D "$source_dir" \
1853         _SYSTEMD_UNIT="testsuite-${TESTID:?}.service" + SYSLOG_IDENTIFIER="testsuite-$TESTID.sh" + \
1854         PRIORITY=4 + PRIORITY=3 + PRIORITY=2 + PRIORITY=1 + PRIORITY=0
1855
1856     if get_bool "$save"; then
1857         # If we don't have systemd-journal-remote copy all journals from /var/log/journal/
1858         # to $dest_dir/journals/ as is, otherwise merge all journals into a single .journal
1859         # file
1860         if [[ -z "${SYSTEMD_JOURNAL_REMOTE:-}" ]]; then
1861             dest="$dest_dir/journals"
1862             mkdir -p "$dest"
1863             cp -a "$source_dir/*" "$dest/"
1864         else
1865             dest="$dest_dir/$dest_name"
1866             "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $source_dir"
1867         fi
1868
1869         if [[ -n "${SUDO_USER:-}" ]]; then
1870             setfacl -R -m "user:$SUDO_USER:r-X" "$dest"
1871         fi
1872
1873         # we want to print this sometime later, so save this in a variable
1874         JOURNAL_LIST="$(ls -lR "$dest")"
1875     fi
1876
1877     rm -rf "${source_dir:?}"/*
1878 }
1879
1880 check_result_common() {
1881     local workspace="${1:?}"
1882     local ret
1883
1884     if [ -s "$workspace/failed" ]; then
1885         # Non-empty …/failed has highest priority
1886         cp -a "$workspace/failed" "${TESTDIR:?}/"
1887         if [ -n "${SUDO_USER}" ]; then
1888             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1889         fi
1890         ret=1
1891     elif get_bool "$TIMED_OUT"; then
1892         echo "(timeout)" >"${TESTDIR:?}/failed"
1893         ret=2
1894     elif [ -e "$workspace/testok" ]; then
1895         # …/testok always counts (but with lower priority than …/failed)
1896         ret=0
1897     elif [ -e "$workspace/skipped" ]; then
1898         # …/skipped always counts (a message is expected)
1899         echo "${TESTNAME:?} was skipped:"
1900         cat "$workspace/skipped"
1901         ret=0
1902     else
1903         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1904         ret=3
1905     fi
1906
1907     check_asan_reports "$workspace" || ret=4
1908
1909     check_coverage_reports "$workspace" || ret=5
1910
1911     save_journal "$workspace/var/log/journal" $ret
1912
1913     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1914         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1915     fi
1916
1917     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1918         echo -n "${TESTNAME:?}: "
1919         cat "$TESTDIR/failed"
1920     fi
1921     echo "${JOURNAL_LIST:-"No journals were saved"}"
1922
1923     return ${ret:?}
1924 }
1925
1926 check_result_nspawn() {
1927     local workspace="${1:?}"
1928     local ret=0
1929
1930     # Run a test-specific checks if defined by check_result_nspawn_hook()
1931     if declare -F check_result_nspawn_hook >/dev/null; then
1932         if ! check_result_nspawn_hook "${workspace}"; then
1933             derror "check_result_nspawn_hook() returned with EC > 0"
1934             ret=4
1935         fi
1936     fi
1937
1938     check_result_common "${workspace}" || ret=$?
1939
1940     _umount_dir "${initdir:?}"
1941
1942     return $ret
1943 }
1944
1945 # can be overridden in specific test
1946 check_result_qemu() {
1947     local ret=0
1948     mount_initdir
1949
1950     # Run a test-specific checks if defined by check_result_qemu_hook()
1951     if declare -F check_result_qemu_hook >/dev/null; then
1952         if ! check_result_qemu_hook "${initdir:?}"; then
1953             derror "check_result_qemu_hook() returned with EC > 0"
1954             ret=4
1955         fi
1956     fi
1957
1958     check_result_common "${initdir:?}" || ret=$?
1959
1960     _umount_dir "${initdir:?}"
1961
1962     return $ret
1963 }
1964
1965 check_result_nspawn_unittests() {
1966     local workspace="${1:?}"
1967     local ret=1
1968
1969     [[ -e "$workspace/testok" ]] && ret=0
1970
1971     if [[ -s "$workspace/failed" ]]; then
1972         ret=$((ret + 1))
1973         echo "=== Failed test log ==="
1974         cat "$workspace/failed"
1975     else
1976         if [[ -s "$workspace/skipped" ]]; then
1977             echo "=== Skipped test log =="
1978             cat "$workspace/skipped"
1979             # We might have only skipped tests - that should not fail the job
1980             ret=0
1981         fi
1982         if [[ -s "$workspace/testok" ]]; then
1983             echo "=== Passed tests ==="
1984             cat "$workspace/testok"
1985         fi
1986     fi
1987
1988     get_bool "${TIMED_OUT:=}" && ret=1
1989     check_coverage_reports "$workspace" || ret=5
1990
1991     save_journal "$workspace/var/log/journal" $ret
1992     echo "${JOURNAL_LIST:-"No journals were saved"}"
1993
1994     _umount_dir "${initdir:?}"
1995
1996     return $ret
1997 }
1998
1999 check_result_qemu_unittests() {
2000     local ret=1
2001
2002     mount_initdir
2003     [[ -e "${initdir:?}/testok" ]] && ret=0
2004
2005     if [[ -s "$initdir/failed" ]]; then
2006         ret=$((ret + 1))
2007         echo "=== Failed test log ==="
2008         cat "$initdir/failed"
2009     else
2010         if [[ -s "$initdir/skipped" ]]; then
2011             echo "=== Skipped test log =="
2012             cat "$initdir/skipped"
2013             # We might have only skipped tests - that should not fail the job
2014             ret=0
2015         fi
2016         if [[ -s "$initdir/testok" ]]; then
2017             echo "=== Passed tests ==="
2018             cat "$initdir/testok"
2019         fi
2020     fi
2021
2022     get_bool "${TIMED_OUT:=}" && ret=1
2023     check_coverage_reports "$initdir" || ret=5
2024
2025     save_journal "$initdir/var/log/journal" $ret
2026     echo "${JOURNAL_LIST:-"No journals were saved"}"
2027
2028     _umount_dir "$initdir"
2029
2030     return $ret
2031 }
2032
2033 create_rc_local() {
2034     dinfo "Create rc.local"
2035     mkdir -p "${initdir:?}/etc/rc.d"
2036     cat >"$initdir/etc/rc.d/rc.local" <<EOF
2037 #!/usr/bin/env bash
2038 exit 0
2039 EOF
2040     chmod 0755 "$initdir/etc/rc.d/rc.local"
2041 }
2042
2043 install_execs() {
2044     ddebug "Install executables from the service files"
2045
2046     local pkg_config_path="${BUILD_DIR:?}/src/core/"
2047     local systemunitdir userunitdir exe
2048     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
2049     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
2050     while read -r exe; do
2051         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
2052         # also, plymouth is pulled in by rescue.service, but even there the exit code
2053         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
2054         dinfo "Attempting to install $exe (based on unit file reference)"
2055         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
2056     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
2057 }
2058
2059 generate_module_dependencies() {
2060     dinfo "Generate modules dependencies"
2061     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
2062         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
2063             dfatal "\"depmod -a $KERNEL_VER\" failed."
2064             exit 1
2065     fi
2066 }
2067
2068 install_depmod_files() {
2069     dinfo "Install depmod files"
2070     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
2071     inst "/lib/modules/$KERNEL_VER/modules.builtin"
2072 }
2073
2074 install_plymouth() {
2075     dinfo "Install plymouth"
2076     # install plymouth, if found... else remove plymouth service files
2077     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
2078     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
2079     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
2080     #         image_install plymouth plymouthd
2081     # else
2082         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
2083     # fi
2084 }
2085
2086 install_haveged() {
2087     # If haveged is installed, it's probably included in initrd and needs to be
2088     # installed in the image too.
2089     if [ -x /usr/sbin/haveged ]; then
2090         dinfo "Install haveged files"
2091         inst /usr/sbin/haveged
2092         for u in /usr/lib/systemd/system/haveged*; do
2093             inst "$u"
2094         done
2095     fi
2096 }
2097
2098 install_ld_so_conf() {
2099     dinfo "Install /etc/ld.so.conf*"
2100     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
2101     ldconfig -r "$initdir"
2102 }
2103
2104 install_testuser() {
2105     dinfo "Set up a test user"
2106     # create unprivileged user for user manager tests
2107     mkdir -p "${initdir:?}/etc/sysusers.d"
2108     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
2109 u testuser    4711     "Test User" /home/testuser /bin/bash
2110 EOF
2111
2112     mkdir -p "$initdir/home/testuser"
2113     chmod 0700 "$initdir/home/testuser"
2114     chown 4711:4711 "$initdir/home/testuser"
2115 }
2116
2117 install_config_files() {
2118     dinfo "Install config files"
2119     inst /etc/sysconfig/init || :
2120     inst /etc/passwd
2121     inst /etc/shadow
2122     inst_any /etc/login.defs /usr/etc/login.defs
2123     inst /etc/group
2124     inst /etc/shells
2125     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
2126     inst /etc/pam.conf || :
2127     inst_any /etc/os-release /usr/lib/os-release
2128     inst /etc/localtime
2129     # we want an empty environment
2130     : >"${initdir:?}/etc/environment"
2131     : >"$initdir/etc/machine-id"
2132     : >"$initdir/etc/resolv.conf"
2133
2134     # set the hostname
2135     echo 'H' >"$initdir/etc/hostname"
2136
2137     # let's set up just one image with the traditional verbose output
2138     if [ "${IMAGE_NAME:?}" != "basic" ]; then
2139         mkdir -p "$initdir/etc/systemd/system.conf.d"
2140         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
2141     fi
2142 }
2143
2144 install_basic_tools() {
2145     dinfo "Install basic tools"
2146     image_install "${BASICTOOLS[@]}"
2147     image_install -o sushell
2148     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
2149     image_install -o ldconfig.real
2150 }
2151
2152 install_debug_tools() {
2153     dinfo "Install debug tools"
2154     image_install -o "${DEBUGTOOLS[@]}"
2155
2156     if get_bool "$INTERACTIVE_DEBUG"; then
2157         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
2158         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
2159         mkdir -p "$getty_override"
2160         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
2161         echo 'export TERM=linux' >>"$initdir/etc/profile"
2162
2163         if command -v resize >/dev/null; then
2164             image_install resize
2165             echo "resize" >>"$initdir/etc/profile"
2166         fi
2167
2168         # Sometimes we might end up with plymouthd still running (especially
2169         # with the initrd -> asan_wrapper -> systemd transition), which will eat
2170         # our inputs and make debugging via tty impossible. Let's fix this by
2171         # killing plymouthd explicitly for the interactive sessions.
2172         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
2173         #   - https://github.com/llvm/llvm-project/issues/49223
2174         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
2175         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
2176         cat >"$plymouth_unit" <<EOF
2177 [Unit]
2178 After=multi-user.target
2179
2180 [Service]
2181 ExecStart=sh -c 'killall --verbose plymouthd || :'
2182
2183 [Install]
2184 WantedBy=multi-user.target
2185 EOF
2186         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
2187     fi
2188 }
2189
2190 install_libnss() {
2191     dinfo "Install libnss"
2192     # install libnss_files for login
2193     local NSS_LIBS
2194     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
2195     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
2196         image_install "${NSS_LIBS[@]}"
2197     fi
2198 }
2199
2200 install_dbus() {
2201     dinfo "Install dbus"
2202     inst "${ROOTLIBDIR:?}/system/dbus.socket"
2203
2204     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2205     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
2206         inst "$ROOTLIBDIR/system/dbus-broker.service"
2207         inst /usr/bin/dbus-broker
2208         inst /usr/bin/dbus-broker-launch
2209         image_install -o {/etc,/usr/lib}/systemd/system/dbus.service
2210     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
2211         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2212         inst "$ROOTLIBDIR/system/dbus-daemon.service"
2213         # Alias symlink
2214         image_install -o {/etc,/usr/lib}/systemd/system/dbus.service
2215     else
2216         inst "$ROOTLIBDIR/system/dbus.service"
2217     fi
2218
2219     while read -r file; do
2220         inst "$file"
2221     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
2222
2223     # setup policy for Type=dbus test
2224     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
2225     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
2226 <?xml version="1.0"?>
2227 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2228         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2229 <busconfig>
2230     <policy user="root">
2231         <allow own="systemd.test.ExecStopPost"/>
2232     </policy>
2233 </busconfig>
2234 EOF
2235
2236     # If we run without KVM, bump the service start timeout
2237     if ! get_bool "$QEMU_KVM"; then
2238         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
2239 <?xml version="1.0"?>
2240 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2241         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2242 <busconfig>
2243     <limit name="service_start_timeout">120000</limit>
2244 </busconfig>
2245 EOF
2246         # Bump the client-side timeout in sd-bus as well
2247         mkdir -p "$initdir/etc/systemd/system.conf.d"
2248         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=120' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
2249     fi
2250 }
2251
2252 install_user_dbus() {
2253     dinfo "Install user dbus"
2254     local userunitdir
2255     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
2256         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
2257         userunitdir=/usr/lib/systemd/user
2258     fi
2259
2260     inst "$userunitdir/dbus.socket"
2261     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
2262
2263     # Append the After= dependency on dbus in case it isn't already set up
2264     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
2265     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
2266 [Unit]
2267 After=dbus.service
2268 EOF
2269
2270     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2271     if [ -f "$userunitdir/dbus-broker.service" ]; then
2272         inst "$userunitdir/dbus-broker.service"
2273         image_install -o {/etc,/usr/lib}/systemd/user/dbus.service
2274     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
2275         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2276         inst "$userunitdir/dbus-daemon.service"
2277         # Alias symlink
2278         image_install -o {/etc,/usr/lib}/systemd/user/dbus.service
2279     else
2280         inst "$userunitdir/dbus.service"
2281     fi
2282 }
2283
2284 install_pam() {
2285     dinfo "Install PAM"
2286     local paths=()
2287
2288     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
2289         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
2290     else
2291         paths+=(/lib*/security)
2292     fi
2293
2294     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
2295         [ -d "$d" ] && paths+=("$d")
2296     done
2297
2298     while read -r file; do
2299         inst "$file"
2300     done < <(find "${paths[@]}" -xtype f)
2301
2302     # pam_unix depends on unix_chkpwd.
2303     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2304     image_install -o unix_chkpwd
2305
2306     # set empty root password for easy debugging
2307     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2308
2309     # And make sure pam_unix will accept it by making sure that
2310     # the PAM module has the nullok option.
2311     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2312         [ -d "$initdir/$d" ] || continue
2313         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2314     done
2315 }
2316
2317 install_locales() {
2318     # install only C.UTF-8 and English locales
2319     dinfo "Install locales"
2320
2321     if command -v meson >/dev/null \
2322             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2323             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2324         # locale-gen support
2325         image_install -o locale-gen localedef
2326         inst /etc/locale.gen || :
2327         inst /usr/share/i18n/SUPPORTED || :
2328         inst_recursive /usr/share/i18n/charmaps
2329         inst_recursive /usr/share/i18n/locales
2330         inst_recursive /usr/share/locale/en*
2331         inst_recursive /usr/share/locale/de*
2332         image_install /usr/share/locale/locale.alias
2333         # locale-gen might either generate each locale separately or merge them
2334         # into a single archive
2335         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2336               image_install /usr/lib/locale/locale-archive); then
2337             dfatal "Failed to install required locales"
2338             exit 1
2339         fi
2340     else
2341         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2342     fi
2343 }
2344
2345 # shellcheck disable=SC2120
2346 install_keymaps() {
2347     local i p
2348     local -a prefix=(
2349         "/usr/lib"
2350         "/usr/share"
2351     )
2352
2353     dinfo "Install console keymaps"
2354
2355     if (( $# == 0 )); then
2356         for p in "${prefix[@]}"; do
2357             # The first three paths may be deprecated.
2358             # It seems now the last three paths are used by many distributions.
2359             for i in \
2360                 "$p"/kbd/keymaps/include/* \
2361                 "$p"/kbd/keymaps/i386/include/* \
2362                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2363                 "$p"/kbd/keymaps/legacy/include/* \
2364                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2365                 "$p"/kbd/keymaps/xkb/us*; do
2366                     [[ -f "$i" ]] || continue
2367                     inst "$i"
2368             done
2369         done
2370     else
2371         # When it takes any argument, then install more keymaps.
2372         for p in "${prefix[@]}"; do
2373             for i in \
2374                 "$p"/kbd/keymaps/include/* \
2375                 "$p"/kbd/keymaps/i386/*/* \
2376                 "$p"/kbd/keymaps/legacy/i386/*/* \
2377                 "$p"/kbd/keymaps/xkb/*; do
2378                     [[ -f "$i" ]] || continue
2379                     inst "$i"
2380             done
2381         done
2382     fi
2383 }
2384
2385 install_x11_keymaps() {
2386     dinfo "Install x11 keymaps"
2387
2388     if (( $# == 0 )); then
2389         # Install only keymap list.
2390         inst /usr/share/X11/xkb/rules/base.lst
2391     else
2392         # When it takes any argument, then install all keymaps.
2393         inst_recursive /usr/share/X11/xkb
2394     fi
2395 }
2396
2397 install_zoneinfo() {
2398     dinfo "Install time zones"
2399     inst_any /usr/share/zoneinfo/Asia/Seoul
2400     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2401     inst_any /usr/share/zoneinfo/Australia/Sydney
2402     inst_any /usr/share/zoneinfo/Europe/Berlin
2403     inst_any /usr/share/zoneinfo/Europe/Dublin
2404     inst_any /usr/share/zoneinfo/Europe/Kiev
2405     inst_any /usr/share/zoneinfo/Pacific/Auckland
2406     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2407     inst_any /usr/share/zoneinfo/CET
2408     inst_any /usr/share/zoneinfo/EET
2409     inst_any /usr/share/zoneinfo/UTC
2410 }
2411
2412 install_fonts() {
2413     dinfo "Install system fonts"
2414     for i in \
2415         /usr/{lib,share}/kbd/consolefonts/eurlatgr* \
2416         /usr/{lib,share}/kbd/consolefonts/latarcyrheb-sun16*; do
2417             [[ -f "$i" ]] || continue
2418             inst "$i"
2419     done
2420 }
2421
2422 install_terminfo() {
2423     dinfo "Install terminfo files"
2424     local terminfodir
2425     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2426         [ -f "${terminfodir}/l/linux" ] && break
2427     done
2428     image_install -o "${terminfodir}/l/linux"
2429 }
2430
2431 has_user_dbus_socket() {
2432     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2433         return 0
2434     else
2435         echo "Per-user instances are not supported. Skipping..."
2436         return 1
2437     fi
2438 }
2439
2440 setup_nspawn_root_hook() { :;}
2441
2442 setup_nspawn_root() {
2443     if [ -z "${initdir}" ]; then
2444         dfatal "\$initdir not defined"
2445         exit 1
2446     fi
2447
2448     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2449
2450     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2451         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2452         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2453     fi
2454
2455     setup_nspawn_root_hook
2456 }
2457
2458 setup_basic_dirs() {
2459     mkdir -p "${initdir:?}/run"
2460     mkdir -p "$initdir/etc/systemd/system"
2461     mkdir -p "$initdir/var/log/journal"
2462
2463
2464     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2465         if [ -L "/$d" ]; then
2466             inst_symlink "/$d"
2467         else
2468             inst_dir "/$d"
2469         fi
2470     done
2471
2472     ln -sfn /run "$initdir/var/run"
2473     ln -sfn /run/lock "$initdir/var/lock"
2474 }
2475
2476 mask_supporting_services() {
2477     # mask some services that we do not want to run in these tests
2478     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2479     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2480     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2481     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2482     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2483 }
2484
2485 inst_libs() {
2486     local bin="${1:?}"
2487     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2488     local file line
2489
2490     while read -r line; do
2491         [[ "$line" = 'not a dynamic executable' ]] && break
2492         # Ignore errors about our own stuff missing. This is most likely caused
2493         # by ldd attempting to use the unprefixed RPATH.
2494         [[ "$line" =~ (libsystemd|libudev).*\ not\ found ]] && continue
2495
2496         if [[ "$line" =~ not\ found ]]; then
2497             dfatal "Missing a shared library required by $bin."
2498             dfatal "Run \"ldd $bin\" to find out what it is."
2499             dfatal "$line"
2500             dfatal "Cannot create a test image."
2501             exit 1
2502         fi
2503
2504         if [[ "$line" =~ $so_regex ]]; then
2505             file="${BASH_REMATCH[1]}"
2506             [[ -e "${initdir:?}/$file" ]] && continue
2507             inst_library "$file"
2508         fi
2509     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2510 }
2511
2512 import_testdir() {
2513     # make sure we don't get a stale LOOPDEV value from old times
2514     local _LOOPDEV="${LOOPDEV:=}"
2515     # We don't want shellcheck to follow & check the $STATEFILE
2516     # shellcheck source=/dev/null
2517     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2518     LOOPDEV="$_LOOPDEV"
2519     if [[ ! -d "$TESTDIR" ]]; then
2520         if [[ -z "$TESTDIR" ]]; then
2521             TESTDIR="$(mktemp --tmpdir=$WORKDIR -d -t systemd-test.XXXXXX)"
2522         else
2523             mkdir -p "$TESTDIR"
2524         fi
2525
2526         cat >"$STATEFILE" <<EOF
2527 TESTDIR="$TESTDIR"
2528 EOF
2529         export TESTDIR
2530     fi
2531
2532     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2533     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2534 }
2535
2536 import_initdir() {
2537     initdir="${TESTDIR:?}/root"
2538     mkdir -p "$initdir"
2539     export initdir
2540 }
2541
2542 get_cgroup_hierarchy() {
2543     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2544         cgroup2fs)
2545             echo "unified"
2546             ;;
2547         tmpfs)
2548             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2549                 echo "hybrid"
2550             else
2551                 echo "legacy"
2552             fi
2553             ;;
2554         *)
2555             dfatal "Failed to determine host's cgroup hierarchy"
2556             exit 1
2557     esac
2558 }
2559
2560 ## @brief Converts numeric logging level to the first letter of level name.
2561 #
2562 # @param lvl Numeric logging level in range from 1 to 6.
2563 # @retval 1 if @a lvl is out of range.
2564 # @retval 0 if @a lvl is correct.
2565 # @result Echoes first letter of level name.
2566 _lvl2char() {
2567     case "$1" in
2568         1) echo F;;
2569         2) echo E;;
2570         3) echo W;;
2571         4) echo I;;
2572         5) echo D;;
2573         6) echo T;;
2574         *) return 1;;
2575     esac
2576 }
2577
2578 ## @brief Internal helper function for _do_dlog()
2579 #
2580 # @param lvl Numeric logging level.
2581 # @param msg Message.
2582 # @retval 0 It's always returned, even if logging failed.
2583 #
2584 # @note This function is not supposed to be called manually. Please use
2585 # dtrace(), ddebug(), or others instead which wrap this one.
2586 #
2587 # This function calls _do_dlog() either with parameter msg, or if
2588 # none is given, it will read standard input and will use every line as
2589 # a message.
2590 #
2591 # This enables:
2592 # dwarn "This is a warning"
2593 # echo "This is a warning" | dwarn
2594 LOG_LEVEL="${LOG_LEVEL:-4}"
2595
2596 dlog() {
2597     local lvl lvlc
2598
2599     [ -z "$LOG_LEVEL" ] && return 0
2600     lvl="${1:?}"; shift
2601     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2602     lvlc="$(_lvl2char "$lvl")" || return 0
2603
2604     if [ $# -ge 1 ]; then
2605         echo "$lvlc: $*"
2606     else
2607         while read -r line; do
2608             echo "$lvlc: " "$line"
2609         done
2610     fi
2611 }
2612
2613 ## @brief Logs message at TRACE level (6)
2614 #
2615 # @param msg Message.
2616 # @retval 0 It's always returned, even if logging failed.
2617 dtrace() {
2618     set +x
2619     dlog 6 "$@"
2620     if get_bool "${debug:=}"; then
2621         set -x
2622     fi
2623 }
2624
2625 ## @brief Logs message at DEBUG level (5)
2626 #
2627 # @param msg Message.
2628 # @retval 0 It's always returned, even if logging failed.
2629 ddebug() {
2630     dlog 5 "$@"
2631 }
2632
2633 ## @brief Logs message at INFO level (4)
2634 #
2635 # @param msg Message.
2636 # @retval 0 It's always returned, even if logging failed.
2637 dinfo() {
2638     set +x
2639     dlog 4 "$@"
2640     if get_bool "${debug:=}"; then
2641         set -x
2642     fi
2643 }
2644
2645 ## @brief Logs message at WARN level (3)
2646 #
2647 # @param msg Message.
2648 # @retval 0 It's always returned, even if logging failed.
2649 dwarn() {
2650     set +x
2651     dlog 3 "$@"
2652     if get_bool "${debug:=}"; then
2653         set -x
2654     fi
2655 }
2656
2657 ## @brief Logs message at ERROR level (2)
2658 #
2659 # @param msg Message.
2660 # @retval 0 It's always returned, even if logging failed.
2661 derror() {
2662     dlog 2 "$@"
2663 }
2664
2665 ## @brief Logs message at FATAL level (1)
2666 #
2667 # @param msg Message.
2668 # @retval 0 It's always returned, even if logging failed.
2669 dfatal() {
2670     set +x
2671     dlog 1 "$@"
2672     if get_bool "${debug:=}"; then
2673         set -x
2674     fi
2675 }
2676
2677
2678 # Generic substring function.  If $2 is in $1, return 0.
2679 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2680
2681 # normalize_path <path>
2682 # Prints the normalized path, where it removes any duplicated
2683 # and trailing slashes.
2684 # Example:
2685 # $ normalize_path ///test/test//
2686 # /test/test
2687 normalize_path() {
2688     shopt -q -s extglob
2689     set -- "${1//+(\/)//}"
2690     shopt -q -u extglob
2691     echo "${1%/}"
2692 }
2693
2694 # convert_abs_rel <from> <to>
2695 # Prints the relative path, when creating a symlink to <to> from <from>.
2696 # Example:
2697 # $ convert_abs_rel /usr/bin/test /bin/test-2
2698 # ../../bin/test-2
2699 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2700 convert_abs_rel() {
2701     local __current __absolute __abssize __cursize __newpath
2702     local -i __i __level
2703
2704     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2705
2706     # corner case #1 - self looping link
2707     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2708
2709     # corner case #2 - own dir link
2710     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2711
2712     IFS="/" read -ra __current <<< "$1"
2713     IFS="/" read -ra __absolute <<< "$2"
2714
2715     __abssize=${#__absolute[@]}
2716     __cursize=${#__current[@]}
2717
2718     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2719         (( __level++ ))
2720         if (( __level > __abssize || __level > __cursize ))
2721         then
2722             break
2723         fi
2724     done
2725
2726     for ((__i = __level; __i < __cursize-1; __i++)); do
2727         if ((__i > __level))
2728         then
2729             __newpath=$__newpath"/"
2730         fi
2731         __newpath=$__newpath".."
2732     done
2733
2734     for ((__i = __level; __i < __abssize; __i++)); do
2735         if [[ -n $__newpath ]]
2736         then
2737             __newpath=$__newpath"/"
2738         fi
2739         __newpath=$__newpath${__absolute[__i]}
2740     done
2741
2742     echo "$__newpath"
2743 }
2744
2745
2746 # Install a directory, keeping symlinks as on the original system.
2747 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2748 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2749 # and a symlink ${initdir}/lib -> lib64.
2750 inst_dir() {
2751     local dir="${1:?}"
2752     local part="${dir%/*}"
2753     local file
2754
2755     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2756
2757     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2758         dir="$part $dir"
2759         part="${part%/*}"
2760     done
2761
2762     # iterate over parent directories
2763     for file in $dir; do
2764         [[ -e "${initdir}/$file" ]] && continue
2765         if [[ -L $file ]]; then
2766             inst_symlink "$file"
2767         else
2768             # create directory
2769             mkdir -m 0755 "${initdir}/$file" || return 1
2770             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2771             chmod u+w "${initdir}/$file"
2772         fi
2773     done
2774 }
2775
2776 # $1 = file to copy to ramdisk
2777 # $2 (optional) Name for the file on the ramdisk
2778 # Location of the image dir is assumed to be $initdir
2779 # We never overwrite the target if it exists.
2780 inst_simple() {
2781     [[ -f "${1:?}" ]] || return 1
2782     strstr "$1" "/" || return 1
2783
2784     local src="$1"
2785     local target="${2:-$1}"
2786     if ! [[ -d ${initdir:?}/$target ]]; then
2787         [[ -e ${initdir}/$target ]] && return 0
2788         [[ -L ${initdir}/$target ]] && return 0
2789         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2790     fi
2791     # install checksum files also
2792     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2793         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2794     fi
2795     ddebug "Installing $src"
2796     cp --sparse=always --force --dereference --preserve=all "$src" "${initdir}/$target"
2797 }
2798
2799 # find symlinks linked to given library file
2800 # $1 = library file
2801 # Function searches for symlinks by stripping version numbers appended to
2802 # library filename, checks if it points to the same target and finally
2803 # prints the list of symlinks to stdout.
2804 #
2805 # Example:
2806 # rev_lib_symlinks libfoo.so.8.1
2807 # output: libfoo.so.8 libfoo.so
2808 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2809 rev_lib_symlinks() {
2810     local fn="${1:?}"
2811     local links=""
2812     local orig
2813     orig="$(readlink -f "$1")"
2814
2815     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2816
2817     until [[ "${fn##*.}" == so ]]; do
2818         fn="${fn%.*}"
2819         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2820     done
2821
2822     echo "${links}"
2823 }
2824
2825 # Same as above, but specialized to handle dynamic libraries.
2826 # It handles making symlinks according to how the original library
2827 # is referenced.
2828 inst_library() {
2829     local src="${1:?}"
2830     local dest="${2:-$1}"
2831     local reallib symlink
2832
2833     strstr "$1" "/" || return 1
2834     [[ -e ${initdir:?}/$dest ]] && return 0
2835     if [[ -L $src ]]; then
2836         # install checksum files also
2837         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2838             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2839         fi
2840         reallib="$(readlink -f "$src")"
2841         inst_simple "$reallib" "$reallib"
2842         inst_dir "${dest%/*}"
2843         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2844         ddebug "Creating symlink $reallib -> $dest"
2845         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2846     else
2847         inst_simple "$src" "$dest"
2848     fi
2849
2850     # Create additional symlinks.  See rev_symlinks description.
2851     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2852         if [[ ! -e "$initdir/$symlink" ]]; then
2853             ddebug "Creating extra symlink: $symlink"
2854             inst_symlink "$symlink"
2855         fi
2856     done
2857 }
2858
2859 # find a binary.  If we were not passed the full path directly,
2860 # search in the usual places to find the binary.
2861 find_binary() {
2862     local bin="${1:?}"
2863     if [[ -z ${bin##/*} ]]; then
2864         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2865             echo "$bin"
2866             return 0
2867         fi
2868     fi
2869
2870     type -P "$bin"
2871 }
2872
2873 # Same as above, but specialized to install binary executables.
2874 # Install binary executable, and all shared library dependencies, if any.
2875 inst_binary() {
2876     local bin="${1:?}"
2877     local path target
2878
2879     # In certain cases we might attempt to install a binary which is already
2880     # present in the test image, yet it's missing from the host system.
2881     # In such cases, let's check if the binary indeed exists in the image
2882     # before doing any other checks. If it does, immediately return with
2883     # success.
2884     if [[ $# -eq 1 ]]; then
2885         for path in "" bin sbin usr/bin usr/sbin; do
2886             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2887         done
2888     fi
2889
2890     bin="$(find_binary "$bin")" || return 1
2891     target="${2:-$bin}"
2892     [[ -e "${initdir:?}/$target" ]] && return 0
2893     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2894
2895     local file line
2896     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2897     # DSOs provided by systemd
2898     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2899     local wrap_binary=0
2900     local enable_lsan=0
2901     # I love bash!
2902     while read -r line; do
2903         [[ "$line" = 'not a dynamic executable' ]] && break
2904
2905         # Ignore errors about our own stuff missing. This is most likely caused
2906         # by ldd attempting to use the unprefixed RPATH.
2907         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2908
2909         # We're built with ASan and the target binary loads one of the systemd's
2910         # DSOs, so we need to tweak the environment before executing the binary
2911         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2912             wrap_binary=1
2913         fi
2914
2915         if [[ "$line" =~ $so_regex ]]; then
2916             file="${BASH_REMATCH[1]}"
2917             [[ -e "${initdir}/$file" ]] && continue
2918             inst_library "$file"
2919             continue
2920         fi
2921
2922         if [[ "$line" =~ not\ found ]]; then
2923             dfatal "Missing a shared library required by $bin."
2924             dfatal "Run \"ldd $bin\" to find out what it is."
2925             dfatal "$line"
2926             dfatal "Cannot create a test image."
2927             exit 1
2928         fi
2929     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2930
2931     # Same as above, but we need to wrap certain libraries unconditionally
2932     #
2933     # chgrp, chown, getent, login, setfacl, su, useradd, userdel
2934     #    - dlopen() (not only) systemd's PAM modules
2935     # ls, mkfs.*, mksquashfs, mkswap, setpriv, stat
2936     #   - pull in nss_systemd with certain options (like ls -l) when
2937     #     nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2938     # delv, dig - pull in nss_resolve if `resolve` is in nsswitch.conf
2939     # tar - called by machinectl in TEST-25
2940     bin_rx='/(agetty|chgrp|chown|curl|delv|dig|getfacl|getent|id|login|ls|mkfs\.[a-z0-9]+|mksquashfs|mkswap|setfacl|setpriv|stat|su|tar|useradd|userdel)$'
2941     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ $bin_rx ]]; then
2942         wrap_binary=1
2943         # Ugh, so we want to disable LSan in most cases for the wrapped binaries, since
2944         # we don't care about memory leaks in such binaries. However, in certain cases
2945         # the external binary is the only interface for the systemd code, like for
2946         # the systemd NSS modules, where we want to detect memory leaks. So let's
2947         # do another check to decide if we want to enable LSan for given binary.
2948         if [[ "$bin" =~ /getent$ ]]; then
2949             enable_lsan=1
2950         fi
2951     fi
2952
2953     # If the target binary is built with ASan support, we don't need to wrap
2954     # it, as it should handle everything by itself
2955     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2956         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2957         # Install the target binary with a ".orig" suffix
2958         inst_simple "$bin" "${target}.orig"
2959         # Create a simple shell wrapper in place of the target binary, which
2960         # sets necessary ASan-related env variables and then exec()s the
2961         # suffixed target binary
2962         cat >"$initdir/$target" <<EOF
2963 #!/bin/bash
2964 # Preload the ASan runtime DSO, otherwise ASAn will complain
2965 export LD_PRELOAD="$ASAN_RT_PATH"
2966 # Disable LSan to speed things up, since we don't care about leak reports
2967 # from 'external' binaries
2968 export ASAN_OPTIONS=detect_leaks=$enable_lsan
2969 # Set argv[0] to the original binary name without the ".orig" suffix
2970 exec -a "\$0" -- "${target}.orig" "\$@"
2971 EOF
2972         chmod +x "$initdir/$target"
2973     else
2974         inst_simple "$bin" "$target"
2975     fi
2976 }
2977
2978 # same as above, except for shell scripts.
2979 # If your shell script does not start with shebang, it is not a shell script.
2980 inst_script() {
2981     local bin line shebang_regex
2982     bin="$(find_binary "${1:?}")" || return 1
2983     shift
2984
2985     read -r -n 80 line <"$bin"
2986     # If debug is set, clean unprintable chars to prevent messing up the term
2987     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2988     shebang_regex='(#! *)(/[^ ]+).*'
2989     [[ "$line" =~ $shebang_regex ]] || return 1
2990     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2991 }
2992
2993 # same as above, but specialized for symlinks
2994 inst_symlink() {
2995     local src="${1:?}"
2996     local target="${2:-$src}"
2997     local realsrc
2998
2999     strstr "$src" "/" || return 1
3000     [[ -L "$src" ]] || return 1
3001     [[ -L "${initdir:?}/$target" ]] && return 0
3002     realsrc="$(readlink -f "$src")"
3003     if ! [[ -e "$initdir/$realsrc" ]]; then
3004         if [[ -d "$realsrc" ]]; then
3005             inst_dir "$realsrc"
3006         else
3007             inst "$realsrc"
3008         fi
3009     fi
3010     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
3011     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
3012     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
3013 }
3014
3015 # attempt to install any programs specified in a udev rule
3016 inst_rule_programs() {
3017     local rule="${1:?}"
3018     local prog bin
3019
3020     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
3021         if [ -x "/lib/udev/$prog" ]; then
3022             bin="/lib/udev/$prog"
3023         else
3024             if ! bin="$(find_binary "$prog")"; then
3025                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
3026                 continue
3027             fi
3028         fi
3029
3030         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
3031         image_install "$bin"
3032     done
3033 }
3034
3035 # udev rules always get installed in the same place, so
3036 # create a function to install them to make life simpler.
3037 inst_rules() {
3038     local target=/etc/udev/rules.d
3039     local found rule
3040
3041     inst_dir "/lib/udev/rules.d"
3042     inst_dir "$target"
3043     for rule in "$@"; do
3044         if [ "${rule#/}" = "$rule" ]; then
3045             for r in /lib/udev/rules.d /etc/udev/rules.d; do
3046                 if [[ -f "$r/$rule" ]]; then
3047                     found="$r/$rule"
3048                     inst_simple "$found"
3049                     inst_rule_programs "$found"
3050                 fi
3051             done
3052         fi
3053         for r in '' ./; do
3054             if [[ -f "${r}${rule}" ]]; then
3055                 found="${r}${rule}"
3056                 inst_simple "$found" "$target/${found##*/}"
3057                 inst_rule_programs "$found"
3058             fi
3059         done
3060         [[ $found ]] || dinfo "Skipping udev rule: $rule"
3061         found=
3062     done
3063 }
3064
3065 # general purpose installation function
3066 # Same args as above.
3067 inst() {
3068     case $# in
3069         1) ;;
3070         2)
3071             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
3072             [[ "$initdir" = "$2" ]] && set "$1"
3073             ;;
3074         3)
3075             [[ -z "$initdir" ]] && export initdir="$2"
3076             set "$1" "$3"
3077             ;;
3078         *)
3079             dfatal "inst only takes 1 or 2 or 3 arguments"
3080             exit 1
3081             ;;
3082     esac
3083
3084     local fun
3085     for fun in inst_symlink inst_script inst_binary inst_simple; do
3086         "$fun" "$@" && return 0
3087     done
3088
3089     dwarn "Failed to install '$1'"
3090     return 1
3091 }
3092
3093 # install any of listed files
3094 #
3095 # If first argument is '-d' and second some destination path, first accessible
3096 # source is installed into this path, otherwise it will installed in the same
3097 # path as source.  If none of listed files was installed, function return 1.
3098 # On first successful installation it returns with 0 status.
3099 #
3100 # Example:
3101 #
3102 # inst_any -d /bin/foo /bin/bar /bin/baz
3103 #
3104 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
3105 # initrd.
3106 inst_any() {
3107     local dest file
3108
3109     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
3110
3111     for file in "$@"; do
3112         if [[ -e "$file" ]]; then
3113             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
3114             inst "$file" && return 0
3115         fi
3116     done
3117
3118     return 1
3119 }
3120
3121 inst_recursive() {
3122     local p item
3123
3124     for p in "$@"; do
3125         # Make sure the source exists, as the process substitution below
3126         # suppresses errors
3127         stat "$p" >/dev/null || return 1
3128
3129         while read -r item; do
3130             if [[ -d "$item" ]]; then
3131                 inst_dir "$item"
3132             elif [[ -f "$item" ]]; then
3133                 inst_simple "$item"
3134             fi
3135         done < <(find "$p" 2>/dev/null)
3136     done
3137 }
3138
3139 # image_install [-o ] <file> [<file> ... ]
3140 # Install <file> to the test image
3141 # -o optionally install the <file> and don't fail, if it is not there
3142 image_install() {
3143     local optional=no
3144     local prog="${1:?}"
3145
3146     if [[ "$prog" = '-o' ]]; then
3147         optional=yes
3148         shift
3149     fi
3150
3151     for prog in "$@"; do
3152         if ! inst "$prog" ; then
3153             if get_bool "$optional"; then
3154                 dinfo "Skipping program $prog as it cannot be found and is" \
3155                     "flagged to be optional"
3156             else
3157                 dfatal "Failed to install $prog"
3158                 exit 1
3159             fi
3160         fi
3161     done
3162 }
3163
3164 # Install a single kernel module along with any firmware it may require.
3165 # $1 = full path to kernel module to install
3166 install_kmod_with_fw() {
3167     local module="${1:?}"
3168     # no need to go further if the module is already installed
3169     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
3170     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
3171
3172     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
3173
3174     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
3175
3176     local modname="${module##*/}"
3177     local fwdir found fw
3178     modname="${modname%.ko*}"
3179
3180     while read -r fw; do
3181         found=
3182         for fwdir in /lib/firmware/updates /lib/firmware; do
3183             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
3184                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
3185                 found=yes
3186             fi
3187         done
3188         if ! get_bool "$found"; then
3189             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
3190                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
3191                     "\"${modname}.ko\""
3192             else
3193                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
3194                     "\"${modname}.ko\""
3195             fi
3196         fi
3197     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
3198     return 0
3199 }
3200
3201 # Do something with all the dependencies of a kernel module.
3202 # Note that kernel modules depend on themselves using the technique we use
3203 # $1 = function to call for each dependency we find
3204 #      It will be passed the full path to the found kernel module
3205 # $2 = module to get dependencies for
3206 # rest of args = arguments to modprobe
3207 for_each_kmod_dep() {
3208     local func="${1:?}"
3209     local kmod="${2:?}"
3210     local found=0
3211     local cmd modpath
3212     shift 2
3213
3214     while read -r cmd modpath _; do
3215         [[ "$cmd" = insmod ]] || continue
3216         "$func" "$modpath" || return $?
3217         found=1
3218     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
3219
3220     ! get_bool "$found" && return 1
3221     return 0
3222 }
3223
3224 # instmods [-c] <kernel module> [<kernel module> ... ]
3225 # instmods [-c] <kernel subsystem>
3226 # install kernel modules along with all their dependencies.
3227 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
3228 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
3229 instmods() {
3230     local check=no
3231     if [[ $# -ge 0 && "$1" = '-c' ]]; then
3232         check=yes
3233         shift
3234     fi
3235
3236     inst1mod() {
3237         local mod="${1:?}"
3238         local ret=0
3239         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
3240
3241         case "$mod" in
3242             =*)
3243                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
3244                     (
3245                         [[ "$mpargs" ]] && echo "$mpargs"
3246                         cat "${mod_dir}/modules.${mod#=}"
3247                     ) | instmods
3248                 else
3249                     (
3250                         [[ "$mpargs" ]] && echo "$mpargs"
3251                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
3252                     ) | instmods
3253                 fi
3254                 ;;
3255             --*)
3256                 mpargs+=" $mod"
3257                 ;;
3258             i2o_scsi)
3259                 # Do not load this diagnostic-only module
3260                 return
3261                 ;;
3262             *)
3263                 mod=${mod##*/}
3264                 # if we are already installed, skip this module and go on
3265                 # to the next one.
3266                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
3267
3268                 # We use '-d' option in modprobe only if modules prefix path
3269                 # differs from default '/'.  This allows us to use Dracut with
3270                 # old version of modprobe which doesn't have '-d' option.
3271                 local mod_dirname=${mod_dir%%/lib/modules/*}
3272                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
3273
3274                 # ok, load the module, all its dependencies, and any firmware
3275                 # it may require
3276                 for_each_kmod_dep install_kmod_with_fw "$mod" \
3277                     --set-version "$KERNEL_VER" \
3278                     ${mod_dirname:+"$mod_dirname"} \
3279                     ${mpargs:+"$mpargs"}
3280                 ((ret+=$?))
3281                 ;;
3282         esac
3283         return "$ret"
3284     }
3285
3286     local mod mpargs
3287
3288     if [[ $# -eq 0 ]]; then  # filenames from stdin
3289         while read -r mod; do
3290             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3291                 dfatal "Failed to install $mod"
3292                 return 1
3293             fi
3294         done
3295     fi
3296
3297     for mod in "$@"; do # filenames as arguments
3298         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3299             dfatal "Failed to install $mod"
3300             return 1
3301         fi
3302     done
3303
3304     return 0
3305 }
3306
3307 _umount_dir() {
3308     local mountpoint="${1:?}"
3309     if mountpoint -q "$mountpoint"; then
3310         ddebug "umount $mountpoint"
3311         umount "$mountpoint"
3312     fi
3313 }
3314
3315 # can be overridden in specific test
3316 test_setup_cleanup() {
3317     cleanup_initdir
3318 }
3319
3320 _test_cleanup() {
3321     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3322     (
3323         set +e
3324         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3325         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3326         # If multiple setups/cleans are ran in parallel, this can cause a race
3327         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3328             rm -vf "${IMAGESTATEDIR}/default.img"
3329         fi
3330         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3331         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3332     ) || :
3333 }
3334
3335 # can be overridden in specific test
3336 test_cleanup() {
3337     _test_cleanup
3338 }
3339
3340 test_cleanup_again() {
3341     [ -n "$TESTDIR" ] || return
3342     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3343     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3344     # Test specific images are not reused, so delete them or we run out of disk space
3345     if [[ -n "$IMAGE_PUBLIC" ]] && [ "$(basename "$IMAGE_PUBLIC")" != "default.img" ]; then
3346         rm -vf "$IMAGE_PUBLIC"
3347     fi
3348     if [[ -n "$IMAGE_PRIVATE" ]] && [ "$(basename "$IMAGE_PRIVATE")" != "default.img" ]; then
3349         rm -vf "$IMAGE_PRIVATE"
3350     fi
3351 }
3352
3353 test_create_image() {
3354     create_empty_image_rootdir
3355
3356     # Create what will eventually be our root filesystem onto an overlay
3357     (
3358         LOG_LEVEL=5
3359         setup_basic_environment
3360     )
3361 }
3362
3363 test_setup() {
3364     if ! get_bool "$NO_BUILD" && \
3365          get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3366          command -v meson >/dev/null && \
3367          [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3368         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3369         exit 1
3370     fi
3371
3372     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3373         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3374         mount_initdir
3375     else
3376         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3377             # default.img is the base that every test uses and optionally appends to
3378             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3379                 # Create the backing public image, but then completely unmount
3380                 # it and drop the loopback device responsible for it, since we're
3381                 # going to symlink/copy the image and mount it again from
3382                 # elsewhere.
3383                 local image_old="${IMAGE_PUBLIC}"
3384                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3385                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3386                 fi
3387                 test_create_image
3388                 test_setup_cleanup
3389                 umount_loopback
3390                 cleanup_loopdev
3391                 IMAGE_PUBLIC="${image_old}"
3392             fi
3393             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3394                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3395             fi
3396         fi
3397
3398         local hook_defined
3399         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3400
3401         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3402         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3403             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3404         else
3405             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3406         fi
3407
3408         mount_initdir
3409
3410         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3411             dinfo "Masking supporting services"
3412             mask_supporting_services
3413         fi
3414
3415         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
3416             # Do an initial coverage capture, to make sure the final report includes
3417             # files that the tests didn't touch at all
3418             lcov --initial --capture --directory "${initdir}/${BUILD_DIR:?}" --output-file "${TESTDIR:?}/coverage-base"
3419         fi
3420
3421         if get_bool "$hook_defined"; then
3422             test_append_files "${initdir:?}"
3423         fi
3424     fi
3425
3426     setup_nspawn_root
3427 }
3428
3429 test_run() {
3430     local test_id="${1:?}"
3431     mount_initdir
3432
3433     if ! get_bool "${TEST_NO_QEMU:=}"; then
3434         if run_qemu "$test_id"; then
3435             check_result_qemu || { echo "qemu test failed"; return 1; }
3436         else
3437             dwarn "can't run qemu, skipping"
3438         fi
3439     fi
3440     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3441         mount_initdir
3442         if run_nspawn "${initdir:?}" "$test_id"; then
3443             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3444         else
3445             dwarn "can't run systemd-nspawn, skipping"
3446         fi
3447
3448         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3449             dir="$TESTDIR/unprivileged-nspawn-root"
3450             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3451                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3452             else
3453                 dwarn "can't run systemd-nspawn, skipping"
3454             fi
3455         fi
3456     fi
3457     return 0
3458 }
3459
3460 do_test() {
3461     if [[ $UID != "0" ]]; then
3462         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3463         exit 0
3464     fi
3465
3466     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3467         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3468         exit 0
3469     fi
3470
3471     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3472         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3473         exit 0
3474     fi
3475
3476     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3477         TEST_NO_QEMU=1
3478     fi
3479
3480     # Detect lib paths
3481     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3482         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3483     done
3484
3485     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3486         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3487     done
3488
3489     mkdir -p "$WORKDIR"
3490     mkdir -p "$STATEDIR"
3491
3492     import_testdir
3493     import_initdir
3494
3495     if [ -n "${SUDO_USER}" ]; then
3496         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3497         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3498     fi
3499
3500     testname="$(basename "$PWD")"
3501
3502     while (($# > 0)); do
3503         case $1 in
3504             --run)
3505                 echo "${testname} RUN: $TEST_DESCRIPTION"
3506                 test_run "$TESTID"
3507                 ret=$?
3508                 if [ $ret -eq 0 ]; then
3509                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3510                 else
3511                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3512                 fi
3513                 exit $ret
3514                 ;;
3515             --setup)
3516                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3517                 test_setup
3518                 test_setup_cleanup
3519                 ;;
3520             --clean)
3521                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3522                 test_cleanup
3523                 ;;
3524             --clean-again)
3525                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3526                 test_cleanup_again
3527                 ;;
3528             --all)
3529                 ret=0
3530                 echo -n "${testname}: $TEST_DESCRIPTION "
3531                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3532                 # and loop devices will leak
3533                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3534                 if [ $ret -eq 0 ]; then
3535                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3536                 fi
3537                 if [ $ret -eq 0 ]; then
3538                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3539                 fi
3540                 test_cleanup
3541                 if [ $ret -eq 0 ]; then
3542                     # $TESTLOG is in $STATEDIR, so clean it up only on success
3543                     [[ -n "$STATEDIR" ]] && rm -vfr "$STATEDIR"
3544                     echo "[OK]"
3545                 else
3546                     echo "[FAILED]"
3547                     echo "see $TESTLOG"
3548                 fi
3549                 exit $ret
3550                 ;;
3551             *)
3552                 break
3553                 ;;
3554         esac
3555         shift
3556     done
3557 }