test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 # Simple wrapper to unify boolean checks.
  18 # Note: this function needs to stay near the top of the file, so we can use it
  19 #       in code in the outermost scope.
  20 get_bool() {
  21     # Make the value lowercase to make the regex matching simpler
  22     local _bool="${1,,}"
  23
  24     # Consider empty value as "false"
  25     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  26         return 1
  27     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  28         return 0
  29     else
  30         echo >&2 "Value '$_bool' is not a valid boolean value"
  31         exit 1
  32     fi
  33 }
  34
  35 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  36 export PATH
  37
  38 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  39 # shellcheck source=/dev/null
  40 source "$os_release"
  41 [[ "$ID" == "debian" || " $ID_LIKE " == *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=no
  42 [[ "$ID" == "arch" || " $ID_LIKE " == *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=no
  43 [[ "$ID" == "fedora" ]] && LOOKS_LIKE_FEDORA=yes || LOOKS_LIKE_FEDORA=no
  44 [[ " $ID_LIKE " == *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=no
  45
  46 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  47 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  48 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  49 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  50 get_bool "$LOOKS_LIKE_SUSE" && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  51 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  52 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  53 # Note that defining a different IMAGE_NAME in a test setup script will only result
  54 # in default.img being copied and renamed. It can then be extended by defining
  55 # a test_append_files() function. The $1 parameter will be the root directory.
  56 # To force creating a new image from scratch (eg: to encrypt it), also define
  57 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  58 IMAGE_NAME=${IMAGE_NAME:-default}
  59 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  60 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  61 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  62 LOOPDEV=
  63
  64 # Since in Bash we can have only one handler per signal, let's overcome this
  65 # limitation by having one global handler for the EXIT signal which executes
  66 # all registered handlers
  67 _AT_EXIT_HANDLERS=()
  68 _at_exit() {
  69     set +e
  70
  71     # Run the EXIT handlers in reverse order
  72     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  73         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  74         eval "${_AT_EXIT_HANDLERS[$i]}"
  75     done
  76 }
  77
  78 trap _at_exit EXIT
  79
  80 add_at_exit_handler() {
  81     _AT_EXIT_HANDLERS+=("${1:?}")
  82 }
  83
  84 # Decide if we can (and want to) run qemu with KVM acceleration.
  85 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  86 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  87 # running under KVM. If these conditions are met, enable KVM (and possibly
  88 # nested KVM), otherwise disable it.
  89 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  90     QEMU_KVM=yes
  91 else
  92     QEMU_KVM=no
  93 fi
  94
  95 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  96     echo "WARNING! Cannot determine libdir from pkg-config, assuming /usr/lib/systemd" >&2
  97     ROOTLIBDIR=/usr/lib/systemd
  98 fi
  99
 100 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
 101 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 102 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 103 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 104 # These variables are used by test scripts
 105 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR
 106
 107 TOOLS_DIR="$SOURCE_DIR/tools"
 108 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 109 if get_bool "${NO_BUILD:=}"; then
 110     BUILD_DIR="$SOURCE_DIR"
 111 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 112     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 113     exit 1
 114 fi
 115
 116 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 117 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 118 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 119 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 120 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 121 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 122 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 123
 124 TESTFILE="${BASH_SOURCE[1]}"
 125 if [ -z "$TESTFILE" ]; then
 126     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 127     exit 1
 128 fi
 129 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 130
 131 WORKDIR="/var/tmp/systemd-tests"
 132 if get_bool "${NO_BUILD:=}"; then
 133     STATEDIR="$WORKDIR/$TESTNAME"
 134 else
 135     STATEDIR="$BUILD_DIR/test/$TESTNAME"
 136 fi
 137
 138 STATEFILE="$STATEDIR/.testdir"
 139 IMAGESTATEDIR="$STATEDIR/.."
 140 TESTLOG="$STATEDIR/test.log"
 141
 142 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 143     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 144     exit 1
 145 fi
 146 TESTID="${BASH_REMATCH[1]:?}"
 147
 148 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 149     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 150     exit 1
 151 fi
 152
 153 BASICTOOLS=(
 154     awk
 155     base64
 156     basename
 157     bash
 158     capsh
 159     cat
 160     chgrp
 161     chmod
 162     chown
 163     chroot
 164     cmp
 165     cp
 166     cryptsetup
 167     cut
 168     date
 169     dd
 170     dhclient
 171     diff
 172     dirname
 173     dmsetup
 174     echo
 175     env
 176     false
 177     find
 178     findmnt
 179     flock
 180     getconf
 181     getent
 182     getfacl
 183     getfattr
 184     grep
 185     grep
 186     gunzip
 187     gzip
 188     head
 189     hostname
 190     id
 191     ionice
 192     ip
 193     jq
 194     killall
 195     ldd
 196     ln
 197     ln
 198     loadkeys
 199     login
 200     losetup
 201     ls
 202     lsattr
 203     lsblk
 204     lz4cat
 205     mkdir
 206     mkfifo
 207     mknod
 208     mktemp
 209     modprobe
 210     mount
 211     mountpoint
 212     mv
 213     nc
 214     nproc
 215     ping
 216     pkill
 217     ps
 218     readlink
 219     realpath
 220     rev
 221     rm
 222     rmdir
 223     rmmod
 224     route
 225     script
 226     sed
 227     seq
 228     setfacl
 229     setfattr
 230     setfont
 231     setpriv
 232     setsid
 233     sfdisk
 234     sh
 235     sleep
 236     sort
 237     stat
 238     stty
 239     su
 240     sulogin
 241     sysctl
 242     tail
 243     tar
 244     tee
 245     test
 246     timeout
 247     touch
 248     tr
 249     true
 250     truncate
 251     tty
 252     umount
 253     uname
 254     unshare
 255     useradd
 256     userdel
 257     wc
 258     xargs
 259     xzcat
 260 )
 261
 262 DEBUGTOOLS=(
 263     df
 264     dmesg
 265     du
 266     free
 267     less
 268     strace
 269     vi
 270     /usr/libexec/vi
 271 )
 272
 273 is_built_with_asan() {
 274     local _bin="${1:?}"
 275
 276     if ! type -P objdump >/dev/null; then
 277         echo "Failed to find objdump, assuming systemd hasn't been built with ASAN."
 278         return 1
 279     fi
 280
 281     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 282     local _asan_calls
 283     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 284     if ((_asan_calls < 1000)); then
 285         return 1
 286     else
 287         return 0
 288     fi
 289 }
 290
 291 is_built_with_coverage() {
 292     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 293         return 1
 294     fi
 295
 296     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 297 }
 298
 299 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 300 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 301
 302 if get_bool "$IS_BUILT_WITH_ASAN"; then
 303     PATH_TO_INIT="$ROOTLIBDIR/systemd-under-asan"
 304     QEMU_MEM="${QEMU_MEM:-2G}"
 305     QEMU_SMP="${QEMU_SMP:-4}"
 306
 307     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 308     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 309         ASAN_COMPILER=gcc
 310         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 311     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 312         ASAN_COMPILER=clang
 313         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 314
 315         # As clang's ASan DSO is usually in a non-standard path, let's check if
 316         # the environment is set accordingly. If not, warn the user and exit.
 317         # We're not setting the LD_LIBRARY_PATH automagically here, because
 318         # user should encounter (and fix) the same issue when running the unit
 319         # tests (meson test)
 320         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 321             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 322             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 323             exit 1
 324         fi
 325     else
 326         echo >&2 "systemd is not linked against the ASan DSO"
 327         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 328         exit 1
 329     fi
 330
 331     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 332 fi
 333
 334 test_require_bin() {
 335     local bin
 336
 337     for bin in "$@"; do
 338         if ! command -v "$bin" >/dev/null; then
 339             echo "Required binary $bin not available, skipping the test"
 340             exit 0
 341         fi
 342     done
 343 }
 344
 345 find_qemu_bin() {
 346     QEMU_BIN="${QEMU_BIN:-""}"
 347     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 348     if get_bool "$QEMU_KVM"; then
 349         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 350     fi
 351
 352     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 353     case $ARCH in
 354     x86_64)
 355         # QEMU's own build system calls it qemu-system-x86_64
 356         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 357         ;;
 358     i*86)
 359         # new i386 version of QEMU
 360         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 361
 362         # i386 version of QEMU
 363         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 364         ;;
 365     ppc64*)
 366         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 367         ;;
 368     esac
 369
 370     if [[ ! -e "$QEMU_BIN" ]]; then
 371         echo "Could not find a suitable qemu binary" >&2
 372         return 1
 373     fi
 374 }
 375
 376 qemu_setup_swtpm_socket() {
 377     local pid state_dir tpm_device
 378
 379     if ! tpm_device="$(qemu_get_tpm_device)"; then
 380         dinfo "Found QEMU version is too old for TPM2 on ppc64le"
 381         exit 0
 382     fi
 383
 384     state_dir="$(mktemp -d)"
 385     swtpm socket --tpm2 --tpmstate dir="$state_dir" --ctrl type=unixio,path="$state_dir/sock" &
 386     pid=$!
 387     if ! kill -0 "$pid"; then
 388         derror "Failed to start swtpm"
 389         return 1
 390     fi
 391
 392     if ! timeout 5 bash -c "until [[ -S $state_dir/sock ]]; do sleep .5; done"; then
 393         derror "Failed to setup swtpm socket"
 394         return 1
 395     fi
 396
 397     dinfo "Started swtpm as PID $pid with state dir $state_dir"
 398
 399     add_at_exit_handler "kill -TERM $pid 2>/dev/null; rm -rf '$state_dir'"
 400
 401     QEMU_OPTIONS+=" -chardev socket,id=chrtpm,path=$state_dir/sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device $tpm_device,tpmdev=tpm0"
 402     dinfo "Configured emulated TPM2 device $tpm_device"
 403
 404     return 0
 405 }
 406
 407 qemu_get_tpm_device() {
 408     local tpm_device="tpm-tis"
 409
 410     if [[ "$(uname -m)" == "ppc64le" ]]; then
 411         # tpm-spapr support was introduced in qemu 5.0.0
 412         if ! qemu_min_version "5.0.0"; then
 413             return 1
 414         fi
 415
 416         tpm_device="tpm-spapr"
 417     fi
 418
 419     echo "$tpm_device"
 420     return 0
 421 }
 422
 423 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 424 # returns 0 if newer or equal
 425 # returns 1 if older
 426 # returns 2 if failing
 427 qemu_min_version() {
 428     find_qemu_bin || return 2
 429
 430     # get version from binary
 431     local qemu_ver
 432     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 433
 434     # Check version string format
 435     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 436     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 437
 438     # compare as last command to return that value
 439     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 440 }
 441
 442 # Pads a file to multiple of 4 bytes
 443 pad4_file() {
 444     local size
 445     size=$(stat -c "%s" "$1")
 446     local padded
 447     padded=$((((size + 3) / 4) * 4))
 448     truncate -s "$padded" "$1"
 449 }
 450
 451 # Return 0 if qemu did run (then you must check the result state/logs for actual
 452 # success), or 1 if qemu is not available.
 453 run_qemu() {
 454     if declare -F run_qemu_hook >/dev/null; then
 455         if ! run_qemu_hook "${workspace}"; then
 456             derror "check_qemu_hook() returned with EC > 0"
 457             ret=4
 458         fi
 459     fi
 460
 461     # If the test provided its own initrd, use it (e.g. TEST-24)
 462     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 463         INITRD="$TESTDIR/initrd.img"
 464     fi
 465
 466     if [ -f /etc/machine-id ]; then
 467         read -r MACHINE_ID </etc/machine-id
 468         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 469             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 470         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 471             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 472     fi
 473
 474     local CONSOLE=ttyS0
 475
 476     # Reset the boot counter, if present
 477     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 478     rm -f "$initdir"/{testok,failed,skipped}
 479     # make sure the initdir is not mounted to avoid concurrent access
 480     cleanup_initdir
 481     umount_loopback
 482
 483     if [[ ! "$KERNEL_BIN" ]]; then
 484         if get_bool "$LOOKS_LIKE_ARCH"; then
 485             KERNEL_BIN=/boot/vmlinuz-linux
 486         else
 487             [ "$ARCH" ] || ARCH=$(uname -m)
 488             case $ARCH in
 489                 ppc64*)
 490                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 491                     # uses the "standard" vmlinuz- prefix
 492                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 493                     CONSOLE=hvc0
 494                     ;;
 495                 *)
 496                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 497                     ;;
 498             esac
 499         fi
 500     fi
 501
 502     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 503     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 504     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 505     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 506     if [[ ! "$INITRD" ]]; then
 507         if [[ -e "$default_fedora_initrd" ]]; then
 508             INITRD="$default_fedora_initrd"
 509         elif get_bool "$LOOKS_LIKE_DEBIAN" && [[ -e "$default_debian_initrd" ]]; then
 510             INITRD="$default_debian_initrd"
 511         elif get_bool "$LOOKS_LIKE_ARCH" && [[ -e "$default_arch_initrd" ]]; then
 512             INITRD="$default_arch_initrd"
 513         elif get_bool "$LOOKS_LIKE_SUSE" && [[ -e "$default_suse_initrd" ]]; then
 514             INITRD="$default_suse_initrd"
 515         fi
 516     fi
 517
 518     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 519     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 520     # is not installed or there's some other error when calling it, fall back
 521     # to the original value (QEMU_SMP=1).
 522     if [[ -z "${QEMU_SMP:=}" ]]; then
 523         if ! QEMU_SMP=$(nproc); then
 524             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 525             QEMU_SMP=1
 526         fi
 527     fi
 528
 529     find_qemu_bin || return 1
 530
 531     if get_bool "${TEST_SETUP_SWTPM:-}"; then
 532         qemu_setup_swtpm_socket || return 1
 533     fi
 534
 535     # Umount initdir to avoid concurrent access to the filesystem
 536     _umount_dir "$initdir"
 537
 538     local kernel_params=()
 539     local qemu_options=()
 540     local qemu_cmd=("$QEMU_BIN")
 541
 542     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 543         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 544     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 545         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 546     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 547         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 548     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 549         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 550         exit 1
 551     fi
 552
 553     if get_bool "$LOOKS_LIKE_SUSE"; then
 554         kernel_params+=("rd.hostonly=0")
 555     fi
 556
 557     # Debian/Ubuntu's initramfs tries to check if it can resume from hibernation
 558     # and wastes a minute or so probing disks, skip that as it's not useful here
 559     kernel_params+=(
 560         "root=LABEL=systemd_boot"
 561         "rw"
 562         "raid=noautodetect"
 563         "rd.luks=0"
 564         "loglevel=2"
 565         "init=$PATH_TO_INIT"
 566         "console=$CONSOLE"
 567         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 568         "systemd.unit=testsuite.target"
 569         "systemd.wants=testsuite-$1.service"
 570         "noresume"
 571         "oops=panic"
 572         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 573         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 574     )
 575
 576     if ! get_bool "$INTERACTIVE_DEBUG" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 577         kernel_params+=(
 578             "panic=1"
 579             "softlockup_panic=1"
 580             "systemd.wants=end.service"
 581         )
 582     fi
 583
 584     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 585     qemu_options+=(
 586         -smp "$QEMU_SMP"
 587         -net none
 588         -m "${QEMU_MEM:-768M}"
 589         -nographic
 590         -kernel "$KERNEL_BIN"
 591         -drive "format=raw,cache=unsafe,file=$image"
 592         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 593     )
 594
 595     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 596         local user_qemu_options
 597         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 598         qemu_options+=("${user_qemu_options[@]}")
 599     fi
 600     qemu_options+=(${QEMU_OPTIONS_ARRAY:+"${QEMU_OPTIONS_ARRAY[@]}"})
 601
 602     if [[ -n "$INITRD" ]]; then
 603         if [[ -n "$INITRD_EXTRA" ]]; then
 604             # An addition initrd has been specified, let's combine it with the main one.
 605             local t="$WORKDIR"/initrd.combined."$RANDOM"
 606
 607             # First, show contents of additional initrd
 608             echo "Additional initrd contents:"
 609             cpio -tv < "$INITRD_EXTRA"
 610
 611             # Copy the main initrd
 612             zstd -d -c -f "$INITRD" > "$t"
 613             add_at_exit_handler "rm $t"
 614             # Kernel requires this to be padded to multiple of 4 bytes with zeroes
 615             pad4_file "$t"
 616
 617             # Copy the additional initrd
 618             cat "$INITRD_EXTRA" >> "$t"
 619             pad4_file "$t"
 620
 621             qemu_options+=(-initrd "$t")
 622         else
 623             qemu_options+=(-initrd "$INITRD")
 624         fi
 625     fi
 626
 627     # Let's use KVM if possible
 628     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 629         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 630     fi
 631
 632     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 633         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 634     fi
 635
 636     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]} ${KERNEL_APPEND:-}" |& tee "${TESTDIR:?}/console.log")
 637     rc=$?
 638     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 639         derror "Test timed out after ${QEMU_TIMEOUT}s"
 640         TIMED_OUT=1
 641     else
 642         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 643     fi
 644     return 0
 645 }
 646
 647 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 648 # success), or 1 if nspawn is not available.
 649 run_nspawn() {
 650     [[ -d /run/systemd/system ]] || return 1
 651     # Reset the boot counter, if present
 652     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 653     rm -f "${initdir:?}"/{testok,failed,skipped}
 654
 655     local nspawn_cmd=()
 656     local nspawn_options=(
 657         "--register=no"
 658         "--kill-signal=SIGKILL"
 659         "--directory=${1:?}"
 660         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 661         "--machine=TEST-$TESTID"
 662     )
 663     local kernel_params=(
 664         "$PATH_TO_INIT"
 665         "systemd.unit=testsuite.target"
 666         "systemd.wants=testsuite-$2.service"
 667         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 668         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 669     )
 670
 671     if get_bool "$INTERACTIVE_DEBUG"; then
 672         nspawn_options+=("--console=interactive")
 673     elif ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 674         kernel_params+=("systemd.wants=end.service")
 675     fi
 676
 677     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 678         local user_nspawn_arguments
 679         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 680         nspawn_options+=("${user_nspawn_arguments[@]}")
 681     fi
 682
 683     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 684         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 685         exit
 686     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 687         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 688     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 689         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 690     else
 691         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 692         exit 1
 693     fi
 694
 695     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 696         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 697     else
 698         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 699     fi
 700
 701     # Word splitting here is intentional
 702     # shellcheck disable=SC2086
 703     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}" ${KERNEL_APPEND:-} |& tee "${TESTDIR:?}/console.log")
 704     rc=$?
 705     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 706         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 707         TIMED_OUT=1
 708     else
 709         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 710     fi
 711     return 0
 712 }
 713
 714 # Build two very minimal root images, with two units, one is the same and one is different across them
 715 install_verity_minimal() {
 716     dinfo "Set up a set of minimal images for verity verification"
 717     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 718         return
 719     fi
 720     if ! command -v mksquashfs >/dev/null 2>&1; then
 721         dfatal "mksquashfs not found"
 722         exit 1
 723     fi
 724     if ! command -v veritysetup >/dev/null 2>&1; then
 725         dfatal "veritysetup not found"
 726         exit 1
 727     fi
 728     # Local modifications of some global variables is intentional in this
 729     # subshell (SC2030)
 730     # shellcheck disable=SC2030
 731     (
 732         BASICTOOLS=(
 733             bash
 734             cat
 735             grep
 736             mount
 737             sleep
 738             touch
 739         )
 740         oldinitdir="$initdir"
 741         rm -rfv "$TESTDIR/minimal"
 742         export initdir="$TESTDIR/minimal"
 743         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 744         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 745         setup_basic_dirs
 746         install_basic_tools
 747         install_ld_so_conf
 748         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 749         # issue, thus falsely triggering SC2030 in this case
 750         # See: koalaman/shellcheck#1409
 751         if [[ -v ASAN_RT_PATH ]]; then
 752             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 753             # into the verity images to get rid of the annoying errors about
 754             # missing $LD_PRELOAD libraries.
 755             inst_libs "$ASAN_RT_PATH"
 756             inst_library "$ASAN_RT_PATH"
 757         fi
 758         cp "$os_release" "$initdir/usr/lib/os-release"
 759         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 760         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 761         touch "$initdir/opt/some_file"
 762         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 763         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 764         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 765 [Service]
 766 ExecStartPre=cat /usr/lib/os-release
 767 ExecStart=sleep 120
 768 EOF
 769         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 770
 771         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 772         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 773             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 774
 775         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 776         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 777         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 778
 779         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 780         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 781             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 782
 783         # Rolling distros like Arch do not set VERSION_ID
 784         local version_id=""
 785         if grep -q "^VERSION_ID=" "$os_release"; then
 786             version_id="$(grep "^VERSION_ID=" "$os_release")"
 787         fi
 788
 789         export initdir="$TESTDIR/app0"
 790         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 791         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 792         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 793         ( echo "${version_id}"
 794           echo "SYSEXT_IMAGE_ID=app" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 795         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 796 [Service]
 797 Type=oneshot
 798 RemainAfterExit=yes
 799 ExecStart=/opt/script0.sh
 800 TemporaryFileSystem=/var/lib
 801 StateDirectory=app0
 802 RuntimeDirectory=app0
 803 EOF
 804         cat >"$initdir/opt/script0.sh" <<EOF
 805 #!/bin/bash
 806 set -e
 807 test -e /usr/lib/os-release
 808 echo bar >\${STATE_DIRECTORY}/foo
 809 cat /usr/lib/extension-release.d/extension-release.app0
 810 EOF
 811         chmod +x "$initdir/opt/script0.sh"
 812         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 813         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 814
 815         export initdir="$TESTDIR/conf0"
 816         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system" "$initdir/opt"
 817         grep "^ID=" "$os_release" >"$initdir/etc/extension-release.d/extension-release.conf0"
 818         echo "${version_id}" >>"$initdir/etc/extension-release.d/extension-release.conf0"
 819         ( echo "${version_id}"
 820           echo "CONFEXT_IMAGE_ID=app" ) >>"$initdir/etc/extension-release.d/extension-release.conf0"
 821         echo MARKER_1 >"$initdir/etc/systemd/system/some_file"
 822         mksquashfs "$initdir" "$oldinitdir/usr/share/conf0.raw" -noappend
 823
 824         export initdir="$TESTDIR/app1"
 825         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 826         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 827         ( echo "${version_id}"
 828           echo "SYSEXT_SCOPE=portable"
 829           echo "SYSEXT_IMAGE_ID=app"
 830           echo "SYSEXT_IMAGE_VERSION=1"
 831           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 832         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 833         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 834 [Service]
 835 Type=oneshot
 836 RemainAfterExit=yes
 837 ExecStart=/opt/script1.sh
 838 StateDirectory=app1
 839 RuntimeDirectory=app1
 840 EOF
 841         cat >"$initdir/opt/script1.sh" <<EOF
 842 #!/bin/bash
 843 set -e
 844 test -e /usr/lib/os-release
 845 echo baz >\${STATE_DIRECTORY}/foo
 846 cat /usr/lib/extension-release.d/extension-release.app2
 847 EOF
 848         chmod +x "$initdir/opt/script1.sh"
 849         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 850         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 851
 852         export initdir="$TESTDIR/app-nodistro"
 853         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 854         ( echo "ID=_any"
 855           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 856         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 857         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 858
 859         export initdir="$TESTDIR/service-scoped-test"
 860         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system"
 861         ( echo "ID=_any"
 862           echo "ARCHITECTURE=_any" ) >"$initdir/etc/extension-release.d/extension-release.service-scoped-test"
 863         echo MARKER_CONFEXT_123 >"$initdir/etc/systemd/system/some_file"
 864         mksquashfs "$initdir" "$oldinitdir/etc/service-scoped-test.raw" -noappend
 865
 866         # We need to create a dedicated sysext image to test the reload mechanism. If we share an image to install the
 867         # 'foo.service' it will be loaded from another test run, which will impact the targeted test.
 868         export initdir="$TESTDIR/app-reload"
 869         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 870         ( echo "ID=_any"
 871           echo "ARCHITECTURE=_any"
 872           echo "EXTENSION_RELOAD_MANAGER=1" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-reload"
 873         mkdir -p "$initdir/usr/lib/systemd/system/multi-user.target.d"
 874         cat >"${initdir}/usr/lib/systemd/system/foo.service" <<EOF
 875 [Service]
 876 Type=oneshot
 877 RemainAfterExit=yes
 878 ExecStart=echo foo
 879
 880 [Install]
 881 WantedBy=multi-user.target
 882 EOF
 883         { echo "[Unit]"; echo "Upholds=foo.service"; } > "$initdir/usr/lib/systemd/system/multi-user.target.d/10-foo-service.conf"
 884         mksquashfs "$initdir" "$oldinitdir/usr/share/app-reload.raw" -noappend
 885     )
 886 }
 887
 888 setup_basic_environment() {
 889     # create the basic filesystem layout
 890     setup_basic_dirs
 891
 892     install_systemd
 893     install_missing_libraries
 894     install_config_files
 895     install_zoneinfo
 896     create_rc_local
 897     install_basic_tools
 898     install_libnss
 899     install_pam
 900     install_dbus
 901     install_fonts
 902     install_locales
 903     install_keymaps
 904     install_x11_keymaps
 905     install_terminfo
 906     install_execs
 907     install_fs_tools
 908     install_modules
 909     install_plymouth
 910     install_haveged
 911     install_debug_tools
 912     install_ld_so_conf
 913     install_testuser
 914     has_user_dbus_socket && install_user_dbus
 915     setup_selinux
 916     install_depmod_files
 917     generate_module_dependencies
 918     if get_bool "$IS_BUILT_WITH_ASAN"; then
 919         create_asan_wrapper
 920     fi
 921     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 922         install_verity_minimal
 923     fi
 924 }
 925
 926 setup_selinux() {
 927     dinfo "Setup SELinux"
 928     # don't forget KERNEL_APPEND='... selinux=1 ...'
 929     if ! get_bool "$SETUP_SELINUX"; then
 930         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 931         return 0
 932     fi
 933
 934     for dir in /etc/selinux /usr/share/selinux; do
 935         rm -rf "${initdir:?}/$dir"
 936         if ! cp -ar "$dir" "$initdir/$dir"; then
 937             dfatal "Failed to copy $dir"
 938             exit 1
 939         fi
 940     done
 941
 942     # We use a custom autorelabel service instead of the SELinux provided set
 943     # of units & a generator, since the generator overrides the default target
 944     # to the SELinux one when it detects /.autorelabel. However, we use
 945     # systemd.unit= on the kernel command cmdline which always takes precedence,
 946     # rendering all SELinux efforts useless. Also, pulling in selinux-autorelabel.service
 947     # explicitly doesn't work either, as it doesn't check for the presence of /.autorelabel
 948     # and does the relabeling unconditionally which always ends with a reboot, so
 949     # we end up in a reboot loop (and it also spews quite a lot of errors as it
 950     # wants /etc/fstab and dracut-initramfs-restore).
 951     touch "$initdir/.autorelabel"
 952     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 953     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 954
 955     # Tools requires by fixfiles
 956     image_install awk bash cat chcon expr egrep find grep head secon setfiles rm sort uname uniq
 957     image_install fixfiles getenforce load_policy selinuxenabled sestatus
 958 }
 959
 960 install_valgrind() {
 961     if ! type -p valgrind; then
 962         dfatal "Failed to install valgrind"
 963         exit 1
 964     fi
 965
 966     local valgrind_bins valgrind_libs valgrind_supp
 967
 968     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null |
 969                                        sed -r -n 's/execve\("([^"]*)".*/\1/p')
 970     image_install "${valgrind_bins[@]}"
 971
 972     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null |
 973                                        sed -r -n 's|.*calling init: (/.*vgpreload_.*)|\1|p')
 974     image_install "${valgrind_libs[@]}"
 975
 976     readarray -t valgrind_supp < <(strace -e open valgrind /bin/true 2>&1 >/dev/null |
 977                                        sed -r -n 's,open\("([^"]*(/debug[^"]*|\.supp))".*= [0-9].*,\1,p')
 978     image_install "${valgrind_supp[@]}"
 979 }
 980
 981 create_valgrind_wrapper() {
 982     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 983     ddebug "Create $valgrind_wrapper"
 984     cat >"$valgrind_wrapper" <<EOF
 985 #!/usr/bin/env bash
 986
 987 mount -t proc proc /proc
 988 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 989 EOF
 990     chmod 0755 "$valgrind_wrapper"
 991 }
 992
 993 create_asan_wrapper() {
 994     local asan_wrapper default_asan_options default_ubsan_options default_environment manager_environment
 995
 996     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 997
 998     default_asan_options="${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}"
 999     default_ubsan_options="${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}"
1000
1001     if [[ "$ASAN_COMPILER" == "clang" ]]; then
1002         # clang: install llvm-symbolizer to generate useful reports
1003         # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
1004         image_install "llvm-symbolizer"
1005
1006         # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
1007         # unnecessary for gcc & libasan, however, for clang this is crucial, as its
1008         # runtime ASan DSO is in a non-standard (library) path.
1009         mkdir -p "${initdir:?}/etc/ld.so.conf.d/"
1010         echo "${ASAN_RT_PATH%/*}" >"${initdir:?}/etc/ld.so.conf.d/asan-path-override.conf"
1011         ldconfig -r "$initdir"
1012     fi
1013
1014     # Create a simple environment file which can be included by systemd services
1015     # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
1016     cat >"${initdir:?}/usr/lib/systemd/systemd-asan-env" <<EOF
1017 LD_PRELOAD=$ASAN_RT_PATH
1018 ASAN_OPTIONS=$default_asan_options
1019 LSAN_OPTIONS=detect_leaks=0
1020 UBSAN_OPTIONS=$default_ubsan_options
1021 EOF
1022
1023     default_environment=(
1024         "ASAN_OPTIONS='$default_asan_options'"
1025         "UBSAN_OPTIONS='$default_ubsan_options'"
1026         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1027     )
1028     manager_environment=(
1029         "ASAN_OPTIONS='$default_asan_options:log_path=/systemd-pid1.asan.log:log_to_syslog=1'"
1030         "UBSAN_OPTIONS='$default_ubsan_options:log_path=/systemd-pid1.ubsan.log:log_to_syslog=1'"
1031         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1032     )
1033
1034     mkdir -p "${initdir:?}/etc/systemd/system.conf.d/"
1035     cat >"${initdir:?}/etc/systemd/system.conf.d/asan.conf" <<EOF
1036 [Manager]
1037 DefaultEnvironment=${default_environment[*]}
1038 ManagerEnvironment=${manager_environment[*]}
1039 DefaultTimeoutStartSec=180s
1040 EOF
1041
1042     # ASAN and syscall filters aren't compatible with each other.
1043     find "${initdir:?}" -name '*.service' -type f -print0 | xargs -0 sed -i 's/^\(MemoryDeny\|SystemCall\)/#\1/'
1044
1045     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journald.service.d/"
1046     cat >"${initdir:?}/etc/systemd/system/systemd-journald.service.d/asan-env.conf" <<EOF
1047 [Service]
1048 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
1049 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
1050 Environment=ASAN_OPTIONS=$default_asan_options:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=$default_ubsan_options:log_path=/systemd-journald.ubsan.log
1051
1052 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
1053 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
1054 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
1055 StandardOutput=file:/systemd-journald.out
1056 EOF
1057
1058     # 90s isn't enough for some services to finish when literally everything is run
1059     # under ASan+UBSan in containers, which, in turn, are run in VMs.
1060     # Let's limit which environments such services should be executed in.
1061     mkdir -p "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/"
1062     cat >"${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/asan.conf" <<EOF
1063 [Unit]
1064 ConditionVirtualization=container
1065
1066 [Service]
1067 TimeoutSec=240s
1068 EOF
1069
1070     # Let's override another hard-coded timeout that kicks in too early
1071     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/"
1072     cat >"${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/asan.conf" <<EOF
1073 [Service]
1074 TimeoutSec=180s
1075 EOF
1076
1077     asan_wrapper="${initdir:?}/${PATH_TO_INIT:?}"
1078     # Sanity check to make sure we don't overwrite something we shouldn't.
1079     [[ "$asan_wrapper" =~ systemd-under-asan$ ]]
1080
1081     cat >"$asan_wrapper" <<EOF
1082 #!/usr/bin/env bash
1083 set -eux
1084
1085 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
1086 export ${manager_environment[@]}
1087 [[ -n "\$ASAN_OPTIONS" && -n "\$UBSAN_OPTIONS" ]]
1088
1089 exec "$ROOTLIBDIR/systemd" "\$@"
1090 EOF
1091     chmod 0755 "$asan_wrapper"
1092 }
1093
1094 create_strace_wrapper() {
1095     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
1096     ddebug "Create $strace_wrapper"
1097     cat >"$strace_wrapper" <<EOF
1098 #!/usr/bin/env bash
1099
1100 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
1101 EOF
1102     chmod 0755 "$strace_wrapper"
1103 }
1104
1105 install_fs_tools() {
1106     dinfo "Install fsck"
1107     image_install /sbin/fsck*
1108     image_install -o /bin/fsck*
1109
1110     # fskc.reiserfs calls reiserfsck. so, install it
1111     image_install -o reiserfsck
1112
1113     # we use mkfs in system-repart tests
1114     image_install /sbin/mkfs.ext4
1115     image_install /sbin/mkfs.vfat
1116 }
1117
1118 install_modules() {
1119     dinfo "Install modules"
1120
1121     instmods bridge dummy ipvlan macvlan vfat veth
1122     instmods loop =block
1123     instmods nls_ascii =nls
1124     instmods overlay =overlayfs
1125     instmods scsi_debug
1126
1127     if get_bool "$LOOKS_LIKE_SUSE"; then
1128         instmods ext4 af_packet
1129     fi
1130 }
1131
1132 install_dmevent() {
1133     instmods dm_crypt =crypto
1134     inst_binary dmeventd
1135     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
1136     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1137         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
1138         # and since buster/bionic 95-dm-notify.rules
1139         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
1140         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
1141     else
1142         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
1143     fi
1144     if get_bool "$LOOKS_LIKE_SUSE"; then
1145         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
1146     fi
1147 }
1148
1149 install_multipath() {
1150     instmods "=md" multipath
1151     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
1152     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
1153     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1154         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
1155     else
1156         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
1157     fi
1158     mkdir -p "${initdir:?}/etc/multipath"
1159
1160     local file
1161     while read -r file; do
1162         # Install libraries required by the given library
1163         inst_libs "$file"
1164         # Install the library itself and create necessary symlinks
1165         inst_library "$file"
1166     done < <(find /lib*/multipath -type f)
1167 }
1168
1169 install_lvm() {
1170     local lvm_rules rule_prefix
1171
1172     image_install lvm
1173     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1174     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1175     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1176
1177     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1178         lvm_rules="56-lvm.rules"
1179         rule_prefix=""
1180     else
1181         lvm_rules="11-dm-lvm.rules"
1182         rule_prefix="dm-"
1183     fi
1184
1185     # Support the new udev autoactivation introduced in lvm 2.03.14
1186     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1187     # Static autoactivation (via lvm2-activation-generator) was dropped
1188     # in lvm 2.03.15
1189     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1190     if [[ -f "/lib/udev/rules.d/69-${rule_prefix}lvm.rules" ]]; then
1191         inst_rules "$lvm_rules" "69-${rule_prefix}lvm.rules"
1192     else
1193         image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1194         image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1195         inst_rules "$lvm_rules" "69-${rule_prefix}lvm-metad.rules"
1196     fi
1197
1198     mkdir -p "${initdir:?}/etc/lvm"
1199 }
1200
1201 install_btrfs() {
1202     instmods btrfs
1203     # Not all utilities provided by btrfs-progs are listed here; extend the list
1204     # if necessary
1205     image_install btrfs btrfstune mkfs.btrfs
1206     inst_rules 64-btrfs-dm.rules
1207 }
1208
1209 install_iscsi() {
1210     # Install both client and server side stuff by default
1211     local inst="${1:-}"
1212     local file
1213
1214     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1215     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1216     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1217         image_install iscsi-iname iscsiadm iscsid iscsistart
1218         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1219         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1220         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1221         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1222         mkdir -p "${initdir:?}/etc/iscsi"
1223         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1224         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1225         # a one-time service instead of distro package's post-install scripts.
1226         # However, some distros still use this approach even after this patch,
1227         # so prefer the already existing initiatorname.iscsi file if it exists.
1228         #
1229         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1230         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1231             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1232             if get_bool "$IS_BUILT_WITH_ASAN"; then
1233                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1234                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1235                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1236                 # (among others) pre-loads ASan's DSO.
1237                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1238                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1239             fi
1240         else
1241             inst_simple "/etc/iscsi/initiatorname.iscsi"
1242         fi
1243     fi
1244
1245     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1246     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1247     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1248         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1249         image_install -o /etc/sysconfig/tgtd
1250         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1251         mkdir -p "${initdir:?}/etc/tgt"
1252         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1253         # Install perl modules required by tgt-admin
1254         #
1255         # Forgive me father for I have sinned. The monstrosity below appends
1256         # a perl snippet to the `tgt-admin` perl script on the fly, which
1257         # dumps a list of files (perl modules) required by `tgt-admin` at
1258         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1259         # passed to `inst_simple` which installs the necessary files into the image
1260         #
1261         # shellcheck disable=SC2016
1262         while read -r file; do
1263             inst_simple "$file"
1264         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1265     fi
1266 }
1267
1268 install_mdadm() {
1269     local unit
1270     local mdadm_units=(
1271         system/mdadm-grow-continue@.service
1272         system/mdadm-last-resort@.service
1273         system/mdadm-last-resort@.timer
1274         system/mdmon@.service
1275         system/mdmonitor-oneshot.service
1276         system/mdmonitor-oneshot.timer
1277         system/mdmonitor.service
1278         system-shutdown/mdadm.shutdown
1279     )
1280
1281     image_install mdadm mdmon
1282     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1283     # Fedora/CentOS/RHEL ships this rule file
1284     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1285
1286     for unit in "${mdadm_units[@]}"; do
1287         image_install "${ROOTLIBDIR:?}/$unit"
1288     done
1289 }
1290
1291 install_compiled_systemd() {
1292     dinfo "Install compiled systemd"
1293
1294     local ninja_bin
1295     ninja_bin="$(type -P ninja || type -P ninja-build)"
1296     if [[ -z "$ninja_bin" ]]; then
1297         dfatal "ninja was not found"
1298         exit 1
1299     fi
1300     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1301
1302     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1303     # find them in the same directory as the runtime data counts
1304     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1305         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1306         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1307         # Set effective & default ACLs for the build dir so unprivileged
1308         # processes can write gcda files with coverage stats
1309         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1310     fi
1311 }
1312
1313 install_package_file() {
1314     local file="${1:?}"
1315
1316     # Skip missing files (like /etc/machine-info)
1317     [[ ! -e "$file" ]] && return 0
1318     # Skip python unit tests, since the image_install machinery will try to pull
1319     # in the whole python stack in a very questionable state, making the tests fail.
1320     # And given we're trying to transition to mkosi-based images anyway I'm not even
1321     # going to bother
1322     [[ "$file" =~ /tests/unit-tests/.*.py$ ]] && return 0
1323     # If the current file is a directory, create it with the original
1324     # mode; if it's a symlink to a directory, copy it as-is
1325     if [[ -d "$file" ]]; then
1326         inst_dir "$file"
1327     else
1328         inst "$file"
1329     fi
1330 }
1331
1332 install_debian_systemd() {
1333     dinfo "Install debian systemd"
1334
1335     local deb file
1336
1337     while read -r deb; do
1338         ddebug "Install debian files from package $deb"
1339         while read -r file; do
1340             install_package_file "$file"
1341         done < <(dpkg-query -L "$deb" 2>/dev/null)
1342     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1343 }
1344
1345 install_rpm() {
1346     local rpm="${1:?}"
1347     local file
1348
1349     if ! rpm -q "$rpm" >/dev/null; then
1350         derror "RPM $rpm is not installed"
1351         return 1
1352     fi
1353
1354     dinfo "Installing contents of RPM $rpm"
1355     while read -r file; do
1356         install_package_file "$file"
1357     done < <(rpm -ql "$rpm")
1358 }
1359
1360 install_suse_systemd() {
1361     local pkgs
1362
1363     dinfo "Install SUSE systemd"
1364
1365     pkgs=(
1366         systemd
1367         systemd-container
1368         systemd-coredump
1369         systemd-experimental
1370         systemd-homed
1371         systemd-journal-remote
1372         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1373         # systemd-network sub-package on openSUSE, has its own testsuite.
1374         systemd-network
1375         systemd-portable
1376         udev
1377     )
1378
1379     for p in "${pkgs[@]}"; do
1380         rpm -q "$p" &>/dev/null || continue
1381
1382         install_rpm "$p"
1383     done
1384
1385     dinfo "Install the data needed by the tests at runtime"
1386     inst_recursive "${SOURCE_DIR}/testdata"
1387     inst_recursive "${SOURCE_DIR}/unit-tests/manual"
1388
1389     # On openSUSE, this directory is not created at package install, at least
1390     # for now.
1391     mkdir -p "$initdir/var/log/journal/remote"
1392 }
1393
1394 install_fedora_systemd() {
1395     local required_packages=(
1396         systemd
1397         systemd-container
1398         systemd-libs
1399         systemd-pam
1400         systemd-tests
1401         systemd-udev
1402     )
1403     local optional_packages=(
1404         systemd-boot-unsigned
1405         systemd-bootchart
1406         systemd-journal-remote
1407         systemd-networkd
1408         systemd-oomd-defaults
1409         systemd-resolved
1410     )
1411     local package
1412
1413     for package in "${required_packages[@]}"; do
1414         install_rpm "$package"
1415     done
1416
1417     for package in "${optional_packages[@]}"; do
1418         rpm -q "$package" >/dev/null || continue
1419         install_rpm "$package"
1420     done
1421 }
1422
1423 install_distro_systemd() {
1424     dinfo "Install distro systemd"
1425
1426     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1427         install_debian_systemd
1428     elif get_bool "$LOOKS_LIKE_SUSE"; then
1429         install_suse_systemd
1430     elif get_bool "$LOOKS_LIKE_FEDORA"; then
1431         install_fedora_systemd
1432     else
1433         dfatal "NO_BUILD not supported for this distro"
1434         exit 1
1435     fi
1436 }
1437
1438 install_systemd() {
1439     dinfo "Install systemd"
1440     if get_bool "$NO_BUILD"; then
1441         install_distro_systemd
1442     else
1443         install_compiled_systemd
1444     fi
1445
1446     # Remove unneeded documentation
1447     rm -fr "${initdir:?}"/usr/share/{man,doc}
1448
1449     # Enable debug logging in PID1
1450     mkdir -p "$initdir/etc/systemd/system.conf.d/"
1451     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/system.conf.d/10-log-level.conf"
1452     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1453         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf.d/99-log-level.conf"
1454     fi
1455     # Enable debug logging for user instances as well
1456     mkdir -p "$initdir/etc/systemd/user.conf.d/"
1457     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/user.conf.d/10-log-level.conf"
1458     # Store coredumps in journal
1459     mkdir -p "$initdir/etc/systemd/coredump.conf.d/"
1460     echo -ne "[Coredump]\nStorage=journal\n" >"$initdir/etc/systemd/coredump.conf.d/10-storage-journal.conf"
1461     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1462     mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
1463     echo -ne "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/99-SYSTEMD_UNIT_PATH.conf"
1464
1465     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1466     # images, since it prevents gcov to write the coverage reports (*.gcda
1467     # files)
1468     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1469         mkdir -p "$initdir/etc/systemd/system/service.d/"
1470         echo -ne "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1471         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image to make the coverage work with
1472         # units using DynamicUser=yes. Do this only for services with test- prefix and a couple of
1473         # known-to-use DynamicUser=yes services, as setting this system-wide has many undesirable
1474         # side-effects, as it creates its own namespace.
1475         for service in test- systemd-journal-{gatewayd,upload}; do
1476             mkdir -p "$initdir/etc/systemd/system/$service.service.d/"
1477             echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/$service.service.d/99-gcov-rwpaths-override.conf"
1478         done
1479         # Ditto, but for the user daemon
1480         mkdir -p "$initdir/etc/systemd/user/test-.service.d/"
1481         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf"
1482         # Bind the $BUILD_DIR into nspawn containers that are executed using
1483         # machinectl. Unfortunately, the .nspawn files don't support drop-ins
1484         # so we have to inject the bind mount directly into
1485         # the systemd-nspawn@.service unit.
1486         cp "$initdir/usr/lib/systemd/system/systemd-nspawn@.service" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1487         sed -ri "s/^ExecStart=.+$/& --bind=${BUILD_DIR//\//\\\/}/" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1488         # Pass the $BUILD_DIR as $COVERAGE_BUILD_DIR env variable to the system
1489         # manager, similarly to what we do with $ASAN_RT_PATH during sanitized
1490         # builds
1491         mkdir -p "$initdir/etc/systemd/system.conf.d/"
1492         echo -ne "[Manager]\nDefaultEnvironment=COVERAGE_BUILD_DIR=$BUILD_DIR\n" >"$initdir/etc/systemd/system.conf.d/99-COVERAGE_BUILD_DIR.conf"
1493     fi
1494
1495     # If we're built with -Dportabled=false, tests with systemd-analyze
1496     # --profile will fail. Since we need just the profile (text) files, let's
1497     # copy them into the image if they don't exist there.
1498     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1499     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1500         dinfo "Couldn't find portable profiles in the test image"
1501         dinfo "Copying them directly from the source tree"
1502         mkdir -p "$portable_dir"
1503         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1504     fi
1505 }
1506
1507 get_ldpath() {
1508     local rpath
1509     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1510
1511     if [ -z "$rpath" ] ; then
1512         echo "$BUILD_DIR"
1513     else
1514         echo "$rpath"
1515     fi
1516 }
1517
1518 install_missing_libraries() {
1519     dinfo "Install missing libraries"
1520     # install possible missing libraries
1521     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/unit-tests/{,manual/,unsafe/}}*; do
1522         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1523     done
1524
1525     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1526     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1527     # if not present
1528     # See: https://github.com/systemd/systemd/pull/23858
1529     while read -r libgcc_s; do
1530         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1531     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1532
1533     local lib path
1534     # A number of dependencies is now optional via dlopen, so the install
1535     # script will not pick them up, since it looks at linkage.
1536     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1; do
1537         ddebug "Searching for $lib via pkg-config"
1538         if pkg-config --exists "$lib"; then
1539                 path="$(pkg-config --variable=libdir "$lib")"
1540                 if [ -z "${path}" ]; then
1541                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1542                     continue
1543                 fi
1544
1545                 if ! [[ ${lib} =~ ^lib ]]; then
1546                         lib="lib${lib}"
1547                 fi
1548                 # p11-kit-1's .so doesn't have the API level in the name
1549                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1550                         lib="libp11-kit"
1551                 fi
1552                 # Some pkg-config files are broken and give out the wrong paths
1553                 # (eg: libcryptsetup), so just ignore them
1554                 inst_libs "${path}/${lib}.so" || true
1555                 inst_library "${path}/${lib}.so" || true
1556
1557                 if [[ "$lib" == "libxkbcommon" ]]; then
1558                     install_x11_keymaps full
1559                 fi
1560         else
1561             ddebug "$lib.pc not found, skipping"
1562             continue
1563         fi
1564     done
1565
1566     # Install extra openssl 3 stuff
1567     path="$(pkg-config --variable=libdir libcrypto)"
1568     inst_simple "${path}/ossl-modules/legacy.so" || true
1569     inst_simple "${path}/ossl-modules/fips.so" || true
1570     inst_simple "${path}/engines-3/afalg.so" || true
1571     inst_simple "${path}/engines-3/capi.so" || true
1572     inst_simple "${path}/engines-3/loader_attic.so" || true
1573     inst_simple "${path}/engines-3/padlock.so" || true
1574
1575     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1576     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1577     # shellcheck disable=SC2046
1578     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1579 }
1580
1581 cleanup_loopdev() {
1582     if [ -n "${LOOPDEV:=}" ]; then
1583         ddebug "losetup -d $LOOPDEV"
1584         losetup -d "${LOOPDEV}"
1585         unset LOOPDEV
1586     fi
1587 }
1588
1589 add_at_exit_handler cleanup_loopdev
1590
1591 create_empty_image() {
1592     if [[ -z "${IMAGE_NAME:=}" ]]; then
1593         echo "create_empty_image: \$IMAGE_NAME not set"
1594         exit 1
1595     fi
1596
1597     # Partition sizes are in MiBs
1598     local root_size=768
1599     local data_size=100
1600     if ! get_bool "$NO_BUILD"; then
1601         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1602             root_size=$((root_size + 200))
1603         fi
1604         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1605             root_size=$((root_size + 200))
1606         fi
1607         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1608             root_size=$((root_size + 250))
1609         fi
1610         if get_bool "$IS_BUILT_WITH_ASAN"; then
1611             root_size=$((root_size * 2))
1612         fi
1613     fi
1614
1615     if [[ "${IMAGE_ADDITIONAL_ROOT_SIZE:-0}" -gt 0 ]]; then
1616         root_size=$((root_size + IMAGE_ADDITIONAL_ROOT_SIZE))
1617     fi
1618     if [[ "${IMAGE_ADDITIONAL_DATA_SIZE:-0}" -gt 0 ]]; then
1619         data_size=$((data_size + IMAGE_ADDITIONAL_DATA_SIZE))
1620     fi
1621
1622     echo "Setting up ${IMAGE_PUBLIC:?} (${root_size} MB)"
1623     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1624
1625     # Create the blank file to use as a root filesystem
1626     truncate -s "${root_size}M" "$IMAGE_PUBLIC"
1627
1628     LOOPDEV="$(losetup --show -P -f "$IMAGE_PUBLIC")"
1629     [[ -b "$LOOPDEV" ]] || return 1
1630     # Create two partitions - a root one and a data one (utilized by some tests)
1631     sfdisk "$LOOPDEV" <<EOF
1632 label: gpt
1633 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=root size=$((root_size - data_size))M bootable
1634 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1635 EOF
1636
1637     udevadm settle
1638
1639     local label=(-L systemd_boot)
1640     # mkfs.reiserfs doesn't know -L. so, use --label instead
1641     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1642     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p1" -q; then
1643         dfatal "Failed to mkfs -t ${FSTYPE}"
1644         exit 1
1645     fi
1646 }
1647
1648 mount_initdir() {
1649     if [ -z "${LOOPDEV:=}" ]; then
1650         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1651         LOOPDEV="$(losetup --show -P -f "$image")"
1652         [ -b "$LOOPDEV" ] || return 1
1653
1654         udevadm settle
1655     fi
1656
1657     if ! mountpoint -q "${initdir:?}"; then
1658         mkdir -p "$initdir"
1659         mount "${LOOPDEV}p1" "$initdir"
1660         TEST_SETUP_CLEANUP_ROOTDIR=1
1661     fi
1662 }
1663
1664 cleanup_initdir() {
1665     # only umount if create_empty_image_rootdir() was called to mount it
1666     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1667         _umount_dir "${initdir:?}"
1668     fi
1669 }
1670
1671 umount_loopback() {
1672     # unmount the loopback device from all places. Otherwise we risk file
1673     # system corruption.
1674     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1675         ddebug "Unmounting all uses of $device"
1676         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1677     done
1678 }
1679
1680 create_empty_image_rootdir() {
1681     create_empty_image
1682     mount_initdir
1683 }
1684
1685 check_asan_reports() {
1686     local ret=0
1687     local root="${1:?}"
1688     local log report
1689
1690     if get_bool "$IS_BUILT_WITH_ASAN"; then
1691         ls -l "$root"
1692         if [[ -e "$root/systemd.asan.log.1" ]]; then
1693             cat "$root/systemd.asan.log.1"
1694             ret=$((ret+1))
1695         fi
1696
1697         for log in pid1 journald; do
1698             report="$(find "$root" -name "systemd-$log.*san.log*" -exec cat {} \;)"
1699             if [[ -n "$report" ]]; then
1700                 printf "%s\n" "$report"
1701                 # shellcheck disable=SC2015
1702                 [[ "$log" == journald ]] && cat "$root/systemd-journald.out" || :
1703                 ret=$((ret+1))
1704             fi
1705         done
1706
1707         # May 08 13:23:31 H testleak[2907148]: SUMMARY: AddressSanitizer: 4 byte(s) leaked in 1 allocation(s).
1708         pids="$(
1709             "$JOURNALCTL" -D "$root/var/log/journal" --grep 'SUMMARY: .*Sanitizer:' |
1710                    grep -v -E 'dbus-daemon|dbus-broker-launch' |
1711                    sed -r -n 's/.* .+\[([0-9]+)\]: SUMMARY:.*/\1/p'
1712         )"
1713
1714         if [[ -n "$pids" ]]; then
1715             ret=$((ret+1))
1716             for pid in $pids; do
1717                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1718             done
1719         fi
1720     fi
1721
1722     return $ret
1723 }
1724
1725 check_coverage_reports() {
1726     local root="${1:?}"
1727
1728     if get_bool "$NO_BUILD"; then
1729         return 0
1730     fi
1731     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1732         return 0
1733     fi
1734
1735     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1736         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1737     else
1738         dest="${TESTDIR:?}/coverage-info"
1739     fi
1740
1741     if [[ ! -e "${TESTDIR:?}/coverage-base" ]]; then
1742         # This shouldn't happen, as the report is generated during the setup
1743         # phase (test_setup()).
1744         derror "Missing base coverage report"
1745         return 1
1746     fi
1747
1748     # Create a coverage report that will later be uploaded. Remove info about system
1749     # libraries/headers and generated files, as we don't really care about them.
1750     lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1751     if [[ -f "$dest" ]]; then
1752         # If the destination report file already exists, don't overwrite it, but
1753         # merge it with the already present one - this usually happens when
1754         # running both "parts" of a test in one run (the qemu and the nspawn part).
1755         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1756     else
1757         # If there's no prior coverage report, merge the new one with the base
1758         # report we did during the setup phase (see test_setup()).
1759         lcov --add-tracefile "${TESTDIR:?}/coverage-base" --add-tracefile "${dest}.new" -o "${dest}"
1760     fi
1761     lcov --remove "$dest" -o "$dest" '/usr/include/*' '/usr/lib/*' "${BUILD_DIR:?}/*"
1762     rm -f "${dest}.new"
1763
1764     # If the test logs contain lines like:
1765     #
1766     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1767     #
1768     # it means we're possibly missing some coverage since gcov can't write the stats,
1769     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1770     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1771     # `setfacl` stuff in install_compiled_systemd().
1772     #
1773     # Also, a note: some tests, like TEST-46, overmount /home with tmpfs, which
1774     # means if your build dir is under /home/your-user (which is usually the
1775     # case) you might get bogus errors and missing coverage.
1776     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1777        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1778         derror "Detected possibly missing coverage, check the journal"
1779         return 1
1780     fi
1781
1782     return 0
1783 }
1784
1785 save_journal() {
1786     local source_dir="${1:?}"
1787     local state="${2:?}"
1788     # Default to always saving journal
1789     local save="yes"
1790     local dest_dir dest_name dest
1791
1792     if [[ "${TEST_SAVE_JOURNAL:-}" == "no" ]]; then
1793         save="no"
1794     elif [[ "${TEST_SAVE_JOURNAL:-}" == "fail" && "$state" -eq 0 ]]; then
1795         save="no"
1796     fi
1797
1798     if [[ -n "${ARTIFACT_DIRECTORY:-}" ]]; then
1799         dest_dir="$ARTIFACT_DIRECTORY"
1800         dest_name="${testname:?}.journal"
1801     else
1802         dest_dir="${TESTDIR:?}"
1803         dest_name="system.journal"
1804     fi
1805
1806     # Show messages from the testsuite-XX.service or messages with priority "warning" and higher
1807     echo " --- $source_dir ---"
1808     "$JOURNALCTL" --no-pager --no-hostname -o short-monotonic -D "$source_dir" \
1809         _SYSTEMD_UNIT="testsuite-${TESTID:?}.service" + PRIORITY=4 + PRIORITY=3 + PRIORITY=2 + PRIORITY=1 + PRIORITY=0
1810
1811     if get_bool "$save"; then
1812         # If we don't have systemd-journal-remote copy all journals from /var/log/journal/
1813         # to $dest_dir/journals/ as is, otherwise merge all journals into a single .journal
1814         # file
1815         if [[ -z "${SYSTEMD_JOURNAL_REMOTE:-}" ]]; then
1816             dest="$dest_dir/journals"
1817             mkdir -p "$dest"
1818             cp -a "$source_dir/*" "$dest/"
1819         else
1820             dest="$dest_dir/$dest_name"
1821             "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $source_dir"
1822         fi
1823
1824         if [[ -n "${SUDO_USER:-}" ]]; then
1825             setfacl -R -m "user:$SUDO_USER:r-X" "$dest"
1826         fi
1827
1828         # we want to print this sometime later, so save this in a variable
1829         JOURNAL_LIST="$(ls -lR "$dest")"
1830     fi
1831
1832     rm -rf "${source_dir:?}"/*
1833 }
1834
1835 check_result_common() {
1836     local workspace="${1:?}"
1837     local ret
1838
1839     if [ -s "$workspace/failed" ]; then
1840         # Non-empty …/failed has highest priority
1841         cp -a "$workspace/failed" "${TESTDIR:?}/"
1842         if [ -n "${SUDO_USER}" ]; then
1843             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1844         fi
1845         ret=1
1846     elif get_bool "$TIMED_OUT"; then
1847         echo "(timeout)" >"${TESTDIR:?}/failed"
1848         ret=2
1849     elif [ -e "$workspace/testok" ]; then
1850         # …/testok always counts (but with lower priority than …/failed)
1851         ret=0
1852     elif [ -e "$workspace/skipped" ]; then
1853         # …/skipped always counts (a message is expected)
1854         echo "${TESTNAME:?} was skipped:"
1855         cat "$workspace/skipped"
1856         ret=0
1857     else
1858         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1859         ret=3
1860     fi
1861
1862     check_asan_reports "$workspace" || ret=4
1863
1864     check_coverage_reports "$workspace" || ret=5
1865
1866     save_journal "$workspace/var/log/journal" $ret
1867
1868     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1869         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1870     fi
1871
1872     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1873         echo -n "${TESTNAME:?}: "
1874         cat "$TESTDIR/failed"
1875     fi
1876     echo "${JOURNAL_LIST:-"No journals were saved"}"
1877
1878     return ${ret:?}
1879 }
1880
1881 check_result_nspawn() {
1882     local workspace="${1:?}"
1883     local ret=0
1884
1885     # Run a test-specific checks if defined by check_result_nspawn_hook()
1886     if declare -F check_result_nspawn_hook >/dev/null; then
1887         if ! check_result_nspawn_hook "${workspace}"; then
1888             derror "check_result_nspawn_hook() returned with EC > 0"
1889             ret=4
1890         fi
1891     fi
1892
1893     check_result_common "${workspace}" || ret=$?
1894
1895     _umount_dir "${initdir:?}"
1896
1897     return $ret
1898 }
1899
1900 # can be overridden in specific test
1901 check_result_qemu() {
1902     local ret=0
1903     mount_initdir
1904
1905     # Run a test-specific checks if defined by check_result_qemu_hook()
1906     if declare -F check_result_qemu_hook >/dev/null; then
1907         if ! check_result_qemu_hook "${initdir:?}"; then
1908             derror "check_result_qemu_hook() returned with EC > 0"
1909             ret=4
1910         fi
1911     fi
1912
1913     check_result_common "${initdir:?}" || ret=$?
1914
1915     _umount_dir "${initdir:?}"
1916
1917     return $ret
1918 }
1919
1920 check_result_nspawn_unittests() {
1921     local workspace="${1:?}"
1922     local ret=1
1923
1924     [[ -e "$workspace/testok" ]] && ret=0
1925
1926     if [[ -s "$workspace/failed" ]]; then
1927         ret=$((ret + 1))
1928         echo "=== Failed test log ==="
1929         cat "$workspace/failed"
1930     else
1931         if [[ -s "$workspace/skipped" ]]; then
1932             echo "=== Skipped test log =="
1933             cat "$workspace/skipped"
1934             # We might have only skipped tests - that should not fail the job
1935             ret=0
1936         fi
1937         if [[ -s "$workspace/testok" ]]; then
1938             echo "=== Passed tests ==="
1939             cat "$workspace/testok"
1940         fi
1941     fi
1942
1943     get_bool "${TIMED_OUT:=}" && ret=1
1944     check_coverage_reports "$workspace" || ret=5
1945
1946     save_journal "$workspace/var/log/journal" $ret
1947     echo "${JOURNAL_LIST:-"No journals were saved"}"
1948
1949     _umount_dir "${initdir:?}"
1950
1951     return $ret
1952 }
1953
1954 check_result_qemu_unittests() {
1955     local ret=1
1956
1957     mount_initdir
1958     [[ -e "${initdir:?}/testok" ]] && ret=0
1959
1960     if [[ -s "$initdir/failed" ]]; then
1961         ret=$((ret + 1))
1962         echo "=== Failed test log ==="
1963         cat "$initdir/failed"
1964     else
1965         if [[ -s "$initdir/skipped" ]]; then
1966             echo "=== Skipped test log =="
1967             cat "$initdir/skipped"
1968             # We might have only skipped tests - that should not fail the job
1969             ret=0
1970         fi
1971         if [[ -s "$initdir/testok" ]]; then
1972             echo "=== Passed tests ==="
1973             cat "$initdir/testok"
1974         fi
1975     fi
1976
1977     get_bool "${TIMED_OUT:=}" && ret=1
1978     check_coverage_reports "$initdir" || ret=5
1979
1980     save_journal "$initdir/var/log/journal" $ret
1981     echo "${JOURNAL_LIST:-"No journals were saved"}"
1982
1983     _umount_dir "$initdir"
1984
1985     return $ret
1986 }
1987
1988 create_rc_local() {
1989     dinfo "Create rc.local"
1990     mkdir -p "${initdir:?}/etc/rc.d"
1991     cat >"$initdir/etc/rc.d/rc.local" <<EOF
1992 #!/usr/bin/env bash
1993 exit 0
1994 EOF
1995     chmod 0755 "$initdir/etc/rc.d/rc.local"
1996 }
1997
1998 install_execs() {
1999     ddebug "Install executables from the service files"
2000
2001     local pkg_config_path="${BUILD_DIR:?}/src/core/"
2002     local systemunitdir userunitdir exe
2003     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
2004     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
2005     while read -r exe; do
2006         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
2007         # also, plymouth is pulled in by rescue.service, but even there the exit code
2008         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
2009         dinfo "Attempting to install $exe (based on unit file reference)"
2010         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
2011     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
2012 }
2013
2014 generate_module_dependencies() {
2015     dinfo "Generate modules dependencies"
2016     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
2017         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
2018             dfatal "\"depmod -a $KERNEL_VER\" failed."
2019             exit 1
2020     fi
2021 }
2022
2023 install_depmod_files() {
2024     dinfo "Install depmod files"
2025     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
2026     inst "/lib/modules/$KERNEL_VER/modules.builtin"
2027 }
2028
2029 install_plymouth() {
2030     dinfo "Install plymouth"
2031     # install plymouth, if found... else remove plymouth service files
2032     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
2033     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
2034     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
2035     #         image_install plymouth plymouthd
2036     # else
2037         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
2038     # fi
2039 }
2040
2041 install_haveged() {
2042     # If haveged is installed, it's probably included in initrd and needs to be
2043     # installed in the image too.
2044     if [ -x /usr/sbin/haveged ]; then
2045         dinfo "Install haveged files"
2046         inst /usr/sbin/haveged
2047         for u in /usr/lib/systemd/system/haveged*; do
2048             inst "$u"
2049         done
2050     fi
2051 }
2052
2053 install_ld_so_conf() {
2054     dinfo "Install /etc/ld.so.conf*"
2055     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
2056     ldconfig -r "$initdir"
2057 }
2058
2059 install_testuser() {
2060     dinfo "Set up a test user"
2061     # create unprivileged user for user manager tests
2062     mkdir -p "${initdir:?}/etc/sysusers.d"
2063     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
2064 u testuser    4711     "Test User" /home/testuser
2065 EOF
2066
2067     mkdir -p "$initdir/home/testuser"
2068     chmod 0700 "$initdir/home/testuser"
2069     chown 4711:4711 "$initdir/home/testuser"
2070 }
2071
2072 install_config_files() {
2073     dinfo "Install config files"
2074     inst /etc/sysconfig/init || :
2075     inst /etc/passwd
2076     inst /etc/shadow
2077     inst_any /etc/login.defs /usr/etc/login.defs
2078     inst /etc/group
2079     inst /etc/shells
2080     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
2081     inst /etc/pam.conf || :
2082     inst_any /etc/os-release /usr/lib/os-release
2083     inst /etc/localtime
2084     # we want an empty environment
2085     : >"${initdir:?}/etc/environment"
2086     : >"$initdir/etc/machine-id"
2087     : >"$initdir/etc/resolv.conf"
2088
2089     # set the hostname
2090     echo 'H' >"$initdir/etc/hostname"
2091
2092     # let's set up just one image with the traditional verbose output
2093     if [ "${IMAGE_NAME:?}" != "basic" ]; then
2094         mkdir -p "$initdir/etc/systemd/system.conf.d"
2095         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
2096     fi
2097 }
2098
2099 install_basic_tools() {
2100     dinfo "Install basic tools"
2101     image_install "${BASICTOOLS[@]}"
2102     image_install -o sushell
2103     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
2104     image_install -o ldconfig.real
2105 }
2106
2107 install_debug_tools() {
2108     dinfo "Install debug tools"
2109     image_install -o "${DEBUGTOOLS[@]}"
2110
2111     if get_bool "$INTERACTIVE_DEBUG"; then
2112         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
2113         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
2114         mkdir -p "$getty_override"
2115         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
2116         echo 'export TERM=linux' >>"$initdir/etc/profile"
2117
2118         if command -v resize >/dev/null; then
2119             image_install resize
2120             echo "resize" >>"$initdir/etc/profile"
2121         fi
2122
2123         # Sometimes we might end up with plymouthd still running (especially
2124         # with the initrd -> asan_wrapper -> systemd transition), which will eat
2125         # our inputs and make debugging via tty impossible. Let's fix this by
2126         # killing plymouthd explicitly for the interactive sessions.
2127         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
2128         #   - https://github.com/llvm/llvm-project/issues/49223
2129         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
2130         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
2131         cat >"$plymouth_unit" <<EOF
2132 [Unit]
2133 After=multi-user.target
2134
2135 [Service]
2136 ExecStart=sh -c 'killall --verbose plymouthd || :'
2137
2138 [Install]
2139 WantedBy=multi-user.target
2140 EOF
2141         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
2142     fi
2143 }
2144
2145 install_libnss() {
2146     dinfo "Install libnss"
2147     # install libnss_files for login
2148     local NSS_LIBS
2149     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
2150     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
2151         image_install "${NSS_LIBS[@]}"
2152     fi
2153 }
2154
2155 install_dbus() {
2156     dinfo "Install dbus"
2157     inst "${ROOTLIBDIR:?}/system/dbus.socket"
2158
2159     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2160     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
2161         inst "$ROOTLIBDIR/system/dbus-broker.service"
2162         inst_symlink /etc/systemd/system/dbus.service
2163         inst /usr/bin/dbus-broker
2164         inst /usr/bin/dbus-broker-launch
2165     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
2166         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2167         inst "$ROOTLIBDIR/system/dbus-daemon.service"
2168         # Alias symlink
2169         inst_symlink /etc/systemd/system/dbus.service
2170     else
2171         inst "$ROOTLIBDIR/system/dbus.service"
2172     fi
2173
2174     while read -r file; do
2175         inst "$file"
2176     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
2177
2178     # setup policy for Type=dbus test
2179     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
2180     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
2181 <?xml version="1.0"?>
2182 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2183         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2184 <busconfig>
2185     <policy user="root">
2186         <allow own="systemd.test.ExecStopPost"/>
2187     </policy>
2188 </busconfig>
2189 EOF
2190
2191     # If we run without KVM, bump the service start timeout
2192     if ! get_bool "$QEMU_KVM"; then
2193         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
2194 <?xml version="1.0"?>
2195 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2196         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2197 <busconfig>
2198     <limit name="service_start_timeout">120000</limit>
2199 </busconfig>
2200 EOF
2201         # Bump the client-side timeout in sd-bus as well
2202         mkdir -p "$initdir/etc/systemd/system.conf.d"
2203         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=120' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
2204     fi
2205 }
2206
2207 install_user_dbus() {
2208     dinfo "Install user dbus"
2209     local userunitdir
2210     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
2211         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
2212         userunitdir=/usr/lib/systemd/user
2213     fi
2214
2215     inst "$userunitdir/dbus.socket"
2216     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
2217
2218     # Append the After= dependency on dbus in case it isn't already set up
2219     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
2220     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
2221 [Unit]
2222 After=dbus.service
2223 EOF
2224
2225     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2226     if [ -f "$userunitdir/dbus-broker.service" ]; then
2227         inst "$userunitdir/dbus-broker.service"
2228         inst_symlink /etc/systemd/user/dbus.service
2229     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
2230         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2231         inst "$userunitdir/dbus-daemon.service"
2232         # Alias symlink
2233         inst_symlink /etc/systemd/user/dbus.service
2234     else
2235         inst "$userunitdir/dbus.service"
2236     fi
2237 }
2238
2239 install_pam() {
2240     dinfo "Install PAM"
2241     local paths=()
2242
2243     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
2244         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
2245     else
2246         paths+=(/lib*/security)
2247     fi
2248
2249     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
2250         [ -d "$d" ] && paths+=("$d")
2251     done
2252
2253     while read -r file; do
2254         inst "$file"
2255     done < <(find "${paths[@]}" -xtype f)
2256
2257     # pam_unix depends on unix_chkpwd.
2258     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2259     image_install -o unix_chkpwd
2260
2261     # set empty root password for easy debugging
2262     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2263
2264     # And make sure pam_unix will accept it by making sure that
2265     # the PAM module has the nullok option.
2266     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2267         [ -d "$initdir/$d" ] || continue
2268         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2269     done
2270 }
2271
2272 install_locales() {
2273     # install only C.UTF-8 and English locales
2274     dinfo "Install locales"
2275
2276     if command -v meson >/dev/null \
2277             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2278             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2279         # locale-gen support
2280         image_install -o locale-gen localedef
2281         inst /etc/locale.gen || :
2282         inst /usr/share/i18n/SUPPORTED || :
2283         inst_recursive /usr/share/i18n/charmaps
2284         inst_recursive /usr/share/i18n/locales
2285         inst_recursive /usr/share/locale/en*
2286         inst_recursive /usr/share/locale/de*
2287         image_install /usr/share/locale/locale.alias
2288         # locale-gen might either generate each locale separately or merge them
2289         # into a single archive
2290         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2291               image_install /usr/lib/locale/locale-archive); then
2292             dfatal "Failed to install required locales"
2293             exit 1
2294         fi
2295     else
2296         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2297     fi
2298 }
2299
2300 # shellcheck disable=SC2120
2301 install_keymaps() {
2302     local i p
2303     local -a prefix=(
2304         "/usr/lib"
2305         "/usr/share"
2306     )
2307
2308     dinfo "Install console keymaps"
2309
2310     if (( $# == 0 )); then
2311         for p in "${prefix[@]}"; do
2312             # The first three paths may be deprecated.
2313             # It seems now the last three paths are used by many distributions.
2314             for i in \
2315                 "$p"/kbd/keymaps/include/* \
2316                 "$p"/kbd/keymaps/i386/include/* \
2317                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2318                 "$p"/kbd/keymaps/legacy/include/* \
2319                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2320                 "$p"/kbd/keymaps/xkb/us*; do
2321                     [[ -f "$i" ]] || continue
2322                     inst "$i"
2323             done
2324         done
2325     else
2326         # When it takes any argument, then install more keymaps.
2327         for p in "${prefix[@]}"; do
2328             for i in \
2329                 "$p"/kbd/keymaps/include/* \
2330                 "$p"/kbd/keymaps/i386/*/* \
2331                 "$p"/kbd/keymaps/legacy/i386/*/* \
2332                 "$p"/kbd/keymaps/xkb/*; do
2333                     [[ -f "$i" ]] || continue
2334                     inst "$i"
2335             done
2336         done
2337     fi
2338 }
2339
2340 install_x11_keymaps() {
2341     dinfo "Install x11 keymaps"
2342
2343     if (( $# == 0 )); then
2344         # Install only keymap list.
2345         inst /usr/share/X11/xkb/rules/base.lst
2346     else
2347         # When it takes any argument, then install all keymaps.
2348         inst_recursive /usr/share/X11/xkb
2349     fi
2350 }
2351
2352 install_zoneinfo() {
2353     dinfo "Install time zones"
2354     inst_any /usr/share/zoneinfo/Asia/Seoul
2355     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2356     inst_any /usr/share/zoneinfo/Australia/Sydney
2357     inst_any /usr/share/zoneinfo/Europe/Berlin
2358     inst_any /usr/share/zoneinfo/Europe/Dublin
2359     inst_any /usr/share/zoneinfo/Europe/Kiev
2360     inst_any /usr/share/zoneinfo/Pacific/Auckland
2361     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2362     inst_any /usr/share/zoneinfo/CET
2363     inst_any /usr/share/zoneinfo/EET
2364     inst_any /usr/share/zoneinfo/UTC
2365 }
2366
2367 install_fonts() {
2368     dinfo "Install system fonts"
2369     for i in \
2370         /usr/{lib,share}/kbd/consolefonts/eurlatgr* \
2371         /usr/{lib,share}/kbd/consolefonts/latarcyrheb-sun16*; do
2372             [[ -f "$i" ]] || continue
2373             inst "$i"
2374     done
2375 }
2376
2377 install_terminfo() {
2378     dinfo "Install terminfo files"
2379     local terminfodir
2380     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2381         [ -f "${terminfodir}/l/linux" ] && break
2382     done
2383     image_install -o "${terminfodir}/l/linux"
2384 }
2385
2386 has_user_dbus_socket() {
2387     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2388         return 0
2389     else
2390         echo "Per-user instances are not supported. Skipping..."
2391         return 1
2392     fi
2393 }
2394
2395 setup_nspawn_root_hook() { :;}
2396
2397 setup_nspawn_root() {
2398     if [ -z "${initdir}" ]; then
2399         dfatal "\$initdir not defined"
2400         exit 1
2401     fi
2402
2403     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2404
2405     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2406         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2407         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2408     fi
2409
2410     setup_nspawn_root_hook
2411 }
2412
2413 setup_basic_dirs() {
2414     mkdir -p "${initdir:?}/run"
2415     mkdir -p "$initdir/etc/systemd/system"
2416     mkdir -p "$initdir/var/log/journal"
2417
2418
2419     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2420         if [ -L "/$d" ]; then
2421             inst_symlink "/$d"
2422         else
2423             inst_dir "/$d"
2424         fi
2425     done
2426
2427     ln -sfn /run "$initdir/var/run"
2428     ln -sfn /run/lock "$initdir/var/lock"
2429 }
2430
2431 mask_supporting_services() {
2432     # mask some services that we do not want to run in these tests
2433     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2434     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2435     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2436     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2437     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2438 }
2439
2440 inst_libs() {
2441     local bin="${1:?}"
2442     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2443     local file line
2444
2445     while read -r line; do
2446         [[ "$line" = 'not a dynamic executable' ]] && break
2447         # Ignore errors about our own stuff missing. This is most likely caused
2448         # by ldd attempting to use the unprefixed RPATH.
2449         [[ "$line" =~ (libsystemd|libudev).*\ not\ found ]] && continue
2450
2451         if [[ "$line" =~ not\ found ]]; then
2452             dfatal "Missing a shared library required by $bin."
2453             dfatal "Run \"ldd $bin\" to find out what it is."
2454             dfatal "$line"
2455             dfatal "Cannot create a test image."
2456             exit 1
2457         fi
2458
2459         if [[ "$line" =~ $so_regex ]]; then
2460             file="${BASH_REMATCH[1]}"
2461             [[ -e "${initdir:?}/$file" ]] && continue
2462             inst_library "$file"
2463         fi
2464     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2465 }
2466
2467 import_testdir() {
2468     # make sure we don't get a stale LOOPDEV value from old times
2469     local _LOOPDEV="${LOOPDEV:=}"
2470     # We don't want shellcheck to follow & check the $STATEFILE
2471     # shellcheck source=/dev/null
2472     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2473     LOOPDEV="$_LOOPDEV"
2474     if [[ ! -d "$TESTDIR" ]]; then
2475         if [[ -z "$TESTDIR" ]]; then
2476             TESTDIR="$(mktemp --tmpdir=$WORKDIR -d -t systemd-test.XXXXXX)"
2477         else
2478             mkdir -p "$TESTDIR"
2479         fi
2480
2481         cat >"$STATEFILE" <<EOF
2482 TESTDIR="$TESTDIR"
2483 EOF
2484         export TESTDIR
2485     fi
2486
2487     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2488     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2489 }
2490
2491 import_initdir() {
2492     initdir="${TESTDIR:?}/root"
2493     mkdir -p "$initdir"
2494     export initdir
2495 }
2496
2497 get_cgroup_hierarchy() {
2498     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2499         cgroup2fs)
2500             echo "unified"
2501             ;;
2502         tmpfs)
2503             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2504                 echo "hybrid"
2505             else
2506                 echo "legacy"
2507             fi
2508             ;;
2509         *)
2510             dfatal "Failed to determine host's cgroup hierarchy"
2511             exit 1
2512     esac
2513 }
2514
2515 ## @brief Converts numeric logging level to the first letter of level name.
2516 #
2517 # @param lvl Numeric logging level in range from 1 to 6.
2518 # @retval 1 if @a lvl is out of range.
2519 # @retval 0 if @a lvl is correct.
2520 # @result Echoes first letter of level name.
2521 _lvl2char() {
2522     case "$1" in
2523         1) echo F;;
2524         2) echo E;;
2525         3) echo W;;
2526         4) echo I;;
2527         5) echo D;;
2528         6) echo T;;
2529         *) return 1;;
2530     esac
2531 }
2532
2533 ## @brief Internal helper function for _do_dlog()
2534 #
2535 # @param lvl Numeric logging level.
2536 # @param msg Message.
2537 # @retval 0 It's always returned, even if logging failed.
2538 #
2539 # @note This function is not supposed to be called manually. Please use
2540 # dtrace(), ddebug(), or others instead which wrap this one.
2541 #
2542 # This function calls _do_dlog() either with parameter msg, or if
2543 # none is given, it will read standard input and will use every line as
2544 # a message.
2545 #
2546 # This enables:
2547 # dwarn "This is a warning"
2548 # echo "This is a warning" | dwarn
2549 LOG_LEVEL="${LOG_LEVEL:-4}"
2550
2551 dlog() {
2552     local lvl lvlc
2553
2554     [ -z "$LOG_LEVEL" ] && return 0
2555     lvl="${1:?}"; shift
2556     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2557     lvlc="$(_lvl2char "$lvl")" || return 0
2558
2559     if [ $# -ge 1 ]; then
2560         echo "$lvlc: $*"
2561     else
2562         while read -r line; do
2563             echo "$lvlc: " "$line"
2564         done
2565     fi
2566 }
2567
2568 ## @brief Logs message at TRACE level (6)
2569 #
2570 # @param msg Message.
2571 # @retval 0 It's always returned, even if logging failed.
2572 dtrace() {
2573     set +x
2574     dlog 6 "$@"
2575     if get_bool "${debug:=}"; then
2576         set -x
2577     fi
2578 }
2579
2580 ## @brief Logs message at DEBUG level (5)
2581 #
2582 # @param msg Message.
2583 # @retval 0 It's always returned, even if logging failed.
2584 ddebug() {
2585     dlog 5 "$@"
2586 }
2587
2588 ## @brief Logs message at INFO level (4)
2589 #
2590 # @param msg Message.
2591 # @retval 0 It's always returned, even if logging failed.
2592 dinfo() {
2593     set +x
2594     dlog 4 "$@"
2595     if get_bool "${debug:=}"; then
2596         set -x
2597     fi
2598 }
2599
2600 ## @brief Logs message at WARN level (3)
2601 #
2602 # @param msg Message.
2603 # @retval 0 It's always returned, even if logging failed.
2604 dwarn() {
2605     set +x
2606     dlog 3 "$@"
2607     if get_bool "${debug:=}"; then
2608         set -x
2609     fi
2610 }
2611
2612 ## @brief Logs message at ERROR level (2)
2613 #
2614 # @param msg Message.
2615 # @retval 0 It's always returned, even if logging failed.
2616 derror() {
2617     dlog 2 "$@"
2618 }
2619
2620 ## @brief Logs message at FATAL level (1)
2621 #
2622 # @param msg Message.
2623 # @retval 0 It's always returned, even if logging failed.
2624 dfatal() {
2625     set +x
2626     dlog 1 "$@"
2627     if get_bool "${debug:=}"; then
2628         set -x
2629     fi
2630 }
2631
2632
2633 # Generic substring function.  If $2 is in $1, return 0.
2634 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2635
2636 # normalize_path <path>
2637 # Prints the normalized path, where it removes any duplicated
2638 # and trailing slashes.
2639 # Example:
2640 # $ normalize_path ///test/test//
2641 # /test/test
2642 normalize_path() {
2643     shopt -q -s extglob
2644     set -- "${1//+(\/)//}"
2645     shopt -q -u extglob
2646     echo "${1%/}"
2647 }
2648
2649 # convert_abs_rel <from> <to>
2650 # Prints the relative path, when creating a symlink to <to> from <from>.
2651 # Example:
2652 # $ convert_abs_rel /usr/bin/test /bin/test-2
2653 # ../../bin/test-2
2654 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2655 convert_abs_rel() {
2656     local __current __absolute __abssize __cursize __newpath
2657     local -i __i __level
2658
2659     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2660
2661     # corner case #1 - self looping link
2662     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2663
2664     # corner case #2 - own dir link
2665     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2666
2667     IFS="/" read -ra __current <<< "$1"
2668     IFS="/" read -ra __absolute <<< "$2"
2669
2670     __abssize=${#__absolute[@]}
2671     __cursize=${#__current[@]}
2672
2673     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2674         (( __level++ ))
2675         if (( __level > __abssize || __level > __cursize ))
2676         then
2677             break
2678         fi
2679     done
2680
2681     for ((__i = __level; __i < __cursize-1; __i++)); do
2682         if ((__i > __level))
2683         then
2684             __newpath=$__newpath"/"
2685         fi
2686         __newpath=$__newpath".."
2687     done
2688
2689     for ((__i = __level; __i < __abssize; __i++)); do
2690         if [[ -n $__newpath ]]
2691         then
2692             __newpath=$__newpath"/"
2693         fi
2694         __newpath=$__newpath${__absolute[__i]}
2695     done
2696
2697     echo "$__newpath"
2698 }
2699
2700
2701 # Install a directory, keeping symlinks as on the original system.
2702 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2703 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2704 # and a symlink ${initdir}/lib -> lib64.
2705 inst_dir() {
2706     local dir="${1:?}"
2707     local part="${dir%/*}"
2708     local file
2709
2710     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2711
2712     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2713         dir="$part $dir"
2714         part="${part%/*}"
2715     done
2716
2717     # iterate over parent directories
2718     for file in $dir; do
2719         [[ -e "${initdir}/$file" ]] && continue
2720         if [[ -L $file ]]; then
2721             inst_symlink "$file"
2722         else
2723             # create directory
2724             mkdir -m 0755 "${initdir}/$file" || return 1
2725             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2726             chmod u+w "${initdir}/$file"
2727         fi
2728     done
2729 }
2730
2731 # $1 = file to copy to ramdisk
2732 # $2 (optional) Name for the file on the ramdisk
2733 # Location of the image dir is assumed to be $initdir
2734 # We never overwrite the target if it exists.
2735 inst_simple() {
2736     [[ -f "${1:?}" ]] || return 1
2737     strstr "$1" "/" || return 1
2738
2739     local src="$1"
2740     local target="${2:-$1}"
2741     if ! [[ -d ${initdir:?}/$target ]]; then
2742         [[ -e ${initdir}/$target ]] && return 0
2743         [[ -L ${initdir}/$target ]] && return 0
2744         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2745     fi
2746     # install checksum files also
2747     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2748         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2749     fi
2750     ddebug "Installing $src"
2751     cp --sparse=always --force --dereference --preserve=all "$src" "${initdir}/$target"
2752 }
2753
2754 # find symlinks linked to given library file
2755 # $1 = library file
2756 # Function searches for symlinks by stripping version numbers appended to
2757 # library filename, checks if it points to the same target and finally
2758 # prints the list of symlinks to stdout.
2759 #
2760 # Example:
2761 # rev_lib_symlinks libfoo.so.8.1
2762 # output: libfoo.so.8 libfoo.so
2763 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2764 rev_lib_symlinks() {
2765     local fn="${1:?}"
2766     local links=""
2767     local orig
2768     orig="$(readlink -f "$1")"
2769
2770     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2771
2772     until [[ "${fn##*.}" == so ]]; do
2773         fn="${fn%.*}"
2774         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2775     done
2776
2777     echo "${links}"
2778 }
2779
2780 # Same as above, but specialized to handle dynamic libraries.
2781 # It handles making symlinks according to how the original library
2782 # is referenced.
2783 inst_library() {
2784     local src="${1:?}"
2785     local dest="${2:-$1}"
2786     local reallib symlink
2787
2788     strstr "$1" "/" || return 1
2789     [[ -e ${initdir:?}/$dest ]] && return 0
2790     if [[ -L $src ]]; then
2791         # install checksum files also
2792         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2793             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2794         fi
2795         reallib="$(readlink -f "$src")"
2796         inst_simple "$reallib" "$reallib"
2797         inst_dir "${dest%/*}"
2798         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2799         ddebug "Creating symlink $reallib -> $dest"
2800         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2801     else
2802         inst_simple "$src" "$dest"
2803     fi
2804
2805     # Create additional symlinks.  See rev_symlinks description.
2806     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2807         if [[ ! -e "$initdir/$symlink" ]]; then
2808             ddebug "Creating extra symlink: $symlink"
2809             inst_symlink "$symlink"
2810         fi
2811     done
2812 }
2813
2814 # find a binary.  If we were not passed the full path directly,
2815 # search in the usual places to find the binary.
2816 find_binary() {
2817     local bin="${1:?}"
2818     if [[ -z ${bin##/*} ]]; then
2819         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2820             echo "$bin"
2821             return 0
2822         fi
2823     fi
2824
2825     type -P "$bin"
2826 }
2827
2828 # Same as above, but specialized to install binary executables.
2829 # Install binary executable, and all shared library dependencies, if any.
2830 inst_binary() {
2831     local bin="${1:?}"
2832     local path target
2833
2834     # In certain cases we might attempt to install a binary which is already
2835     # present in the test image, yet it's missing from the host system.
2836     # In such cases, let's check if the binary indeed exists in the image
2837     # before doing any other checks. If it does, immediately return with
2838     # success.
2839     if [[ $# -eq 1 ]]; then
2840         for path in "" bin sbin usr/bin usr/sbin; do
2841             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2842         done
2843     fi
2844
2845     bin="$(find_binary "$bin")" || return 1
2846     target="${2:-$bin}"
2847     [[ -e "${initdir:?}/$target" ]] && return 0
2848     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2849
2850     local file line
2851     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2852     # DSOs provided by systemd
2853     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2854     local wrap_binary=0
2855     local enable_lsan=0
2856     # I love bash!
2857     while read -r line; do
2858         [[ "$line" = 'not a dynamic executable' ]] && break
2859
2860         # Ignore errors about our own stuff missing. This is most likely caused
2861         # by ldd attempting to use the unprefixed RPATH.
2862         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2863
2864         # We're built with ASan and the target binary loads one of the systemd's
2865         # DSOs, so we need to tweak the environment before executing the binary
2866         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2867             wrap_binary=1
2868         fi
2869
2870         if [[ "$line" =~ $so_regex ]]; then
2871             file="${BASH_REMATCH[1]}"
2872             [[ -e "${initdir}/$file" ]] && continue
2873             inst_library "$file"
2874             continue
2875         fi
2876
2877         if [[ "$line" =~ not\ found ]]; then
2878             dfatal "Missing a shared library required by $bin."
2879             dfatal "Run \"ldd $bin\" to find out what it is."
2880             dfatal "$line"
2881             dfatal "Cannot create a test image."
2882             exit 1
2883         fi
2884     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2885
2886     # Same as above, but we need to wrap certain libraries unconditionally
2887     #
2888     # chgrp, chown, getent, login, setfacl, su, useradd, userdel
2889     #    - dlopen() (not only) systemd's PAM modules
2890     # ls, mkfs.*, mksquashfs, mkswap, setpriv, stat
2891     #   - pull in nss_systemd with certain options (like ls -l) when
2892     #     nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2893     # delv, dig - pull in nss_resolve if `resolve` is in nsswitch.conf
2894     # tar - called by machinectl in TEST-25
2895     bin_rx='/(agetty|chgrp|chown|curl|delv|dig|getfacl|getent|id|login|ls|mkfs\.[a-z0-9]+|mksquashfs|mkswap|setfacl|setpriv|stat|su|tar|useradd|userdel)$'
2896     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ $bin_rx ]]; then
2897         wrap_binary=1
2898         # Ugh, so we want to disable LSan in most cases for the wrapped binaries, since
2899         # we don't care about memory leaks in such binaries. However, in certain cases
2900         # the external binary is the only interface for the systemd code, like for
2901         # the systemd NSS modules, where we want to detect memory leaks. So let's
2902         # do another check to decide if we want to enable LSan for given binary.
2903         if [[ "$bin" =~ /getent$ ]]; then
2904             enable_lsan=1
2905         fi
2906     fi
2907
2908     # If the target binary is built with ASan support, we don't need to wrap
2909     # it, as it should handle everything by itself
2910     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2911         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2912         # Install the target binary with a ".orig" suffix
2913         inst_simple "$bin" "${target}.orig"
2914         # Create a simple shell wrapper in place of the target binary, which
2915         # sets necessary ASan-related env variables and then exec()s the
2916         # suffixed target binary
2917         cat >"$initdir/$target" <<EOF
2918 #!/bin/bash
2919 # Preload the ASan runtime DSO, otherwise ASAn will complain
2920 export LD_PRELOAD="$ASAN_RT_PATH"
2921 # Disable LSan to speed things up, since we don't care about leak reports
2922 # from 'external' binaries
2923 export ASAN_OPTIONS=detect_leaks=$enable_lsan
2924 # Set argv[0] to the original binary name without the ".orig" suffix
2925 exec -a "\$0" -- "${target}.orig" "\$@"
2926 EOF
2927         chmod +x "$initdir/$target"
2928     else
2929         inst_simple "$bin" "$target"
2930     fi
2931 }
2932
2933 # same as above, except for shell scripts.
2934 # If your shell script does not start with shebang, it is not a shell script.
2935 inst_script() {
2936     local bin line shebang_regex
2937     bin="$(find_binary "${1:?}")" || return 1
2938     shift
2939
2940     read -r -n 80 line <"$bin"
2941     # If debug is set, clean unprintable chars to prevent messing up the term
2942     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2943     shebang_regex='(#! *)(/[^ ]+).*'
2944     [[ "$line" =~ $shebang_regex ]] || return 1
2945     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2946 }
2947
2948 # same as above, but specialized for symlinks
2949 inst_symlink() {
2950     local src="${1:?}"
2951     local target="${2:-$src}"
2952     local realsrc
2953
2954     strstr "$src" "/" || return 1
2955     [[ -L "$src" ]] || return 1
2956     [[ -L "${initdir:?}/$target" ]] && return 0
2957     realsrc="$(readlink -f "$src")"
2958     if ! [[ -e "$initdir/$realsrc" ]]; then
2959         if [[ -d "$realsrc" ]]; then
2960             inst_dir "$realsrc"
2961         else
2962             inst "$realsrc"
2963         fi
2964     fi
2965     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
2966     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
2967     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
2968 }
2969
2970 # attempt to install any programs specified in a udev rule
2971 inst_rule_programs() {
2972     local rule="${1:?}"
2973     local prog bin
2974
2975     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
2976         if [ -x "/lib/udev/$prog" ]; then
2977             bin="/lib/udev/$prog"
2978         else
2979             if ! bin="$(find_binary "$prog")"; then
2980                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
2981                 continue
2982             fi
2983         fi
2984
2985         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
2986         image_install "$bin"
2987     done
2988 }
2989
2990 # udev rules always get installed in the same place, so
2991 # create a function to install them to make life simpler.
2992 inst_rules() {
2993     local target=/etc/udev/rules.d
2994     local found rule
2995
2996     inst_dir "/lib/udev/rules.d"
2997     inst_dir "$target"
2998     for rule in "$@"; do
2999         if [ "${rule#/}" = "$rule" ]; then
3000             for r in /lib/udev/rules.d /etc/udev/rules.d; do
3001                 if [[ -f "$r/$rule" ]]; then
3002                     found="$r/$rule"
3003                     inst_simple "$found"
3004                     inst_rule_programs "$found"
3005                 fi
3006             done
3007         fi
3008         for r in '' ./; do
3009             if [[ -f "${r}${rule}" ]]; then
3010                 found="${r}${rule}"
3011                 inst_simple "$found" "$target/${found##*/}"
3012                 inst_rule_programs "$found"
3013             fi
3014         done
3015         [[ $found ]] || dinfo "Skipping udev rule: $rule"
3016         found=
3017     done
3018 }
3019
3020 # general purpose installation function
3021 # Same args as above.
3022 inst() {
3023     case $# in
3024         1) ;;
3025         2)
3026             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
3027             [[ "$initdir" = "$2" ]] && set "$1"
3028             ;;
3029         3)
3030             [[ -z "$initdir" ]] && export initdir="$2"
3031             set "$1" "$3"
3032             ;;
3033         *)
3034             dfatal "inst only takes 1 or 2 or 3 arguments"
3035             exit 1
3036             ;;
3037     esac
3038
3039     local fun
3040     for fun in inst_symlink inst_script inst_binary inst_simple; do
3041         "$fun" "$@" && return 0
3042     done
3043
3044     dwarn "Failed to install '$1'"
3045     return 1
3046 }
3047
3048 # install any of listed files
3049 #
3050 # If first argument is '-d' and second some destination path, first accessible
3051 # source is installed into this path, otherwise it will installed in the same
3052 # path as source.  If none of listed files was installed, function return 1.
3053 # On first successful installation it returns with 0 status.
3054 #
3055 # Example:
3056 #
3057 # inst_any -d /bin/foo /bin/bar /bin/baz
3058 #
3059 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
3060 # initrd.
3061 inst_any() {
3062     local dest file
3063
3064     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
3065
3066     for file in "$@"; do
3067         if [[ -e "$file" ]]; then
3068             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
3069             inst "$file" && return 0
3070         fi
3071     done
3072
3073     return 1
3074 }
3075
3076 inst_recursive() {
3077     local p item
3078
3079     for p in "$@"; do
3080         # Make sure the source exists, as the process substitution below
3081         # suppresses errors
3082         stat "$p" >/dev/null || return 1
3083
3084         while read -r item; do
3085             if [[ -d "$item" ]]; then
3086                 inst_dir "$item"
3087             elif [[ -f "$item" ]]; then
3088                 inst_simple "$item"
3089             fi
3090         done < <(find "$p" 2>/dev/null)
3091     done
3092 }
3093
3094 # image_install [-o ] <file> [<file> ... ]
3095 # Install <file> to the test image
3096 # -o optionally install the <file> and don't fail, if it is not there
3097 image_install() {
3098     local optional=no
3099     local prog="${1:?}"
3100
3101     if [[ "$prog" = '-o' ]]; then
3102         optional=yes
3103         shift
3104     fi
3105
3106     for prog in "$@"; do
3107         if ! inst "$prog" ; then
3108             if get_bool "$optional"; then
3109                 dinfo "Skipping program $prog as it cannot be found and is" \
3110                     "flagged to be optional"
3111             else
3112                 dfatal "Failed to install $prog"
3113                 exit 1
3114             fi
3115         fi
3116     done
3117 }
3118
3119 # Install a single kernel module along with any firmware it may require.
3120 # $1 = full path to kernel module to install
3121 install_kmod_with_fw() {
3122     local module="${1:?}"
3123     # no need to go further if the module is already installed
3124     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
3125     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
3126
3127     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
3128
3129     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
3130
3131     local modname="${module##*/}"
3132     local fwdir found fw
3133     modname="${modname%.ko*}"
3134
3135     while read -r fw; do
3136         found=
3137         for fwdir in /lib/firmware/updates /lib/firmware; do
3138             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
3139                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
3140                 found=yes
3141             fi
3142         done
3143         if ! get_bool "$found"; then
3144             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
3145                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
3146                     "\"${modname}.ko\""
3147             else
3148                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
3149                     "\"${modname}.ko\""
3150             fi
3151         fi
3152     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
3153     return 0
3154 }
3155
3156 # Do something with all the dependencies of a kernel module.
3157 # Note that kernel modules depend on themselves using the technique we use
3158 # $1 = function to call for each dependency we find
3159 #      It will be passed the full path to the found kernel module
3160 # $2 = module to get dependencies for
3161 # rest of args = arguments to modprobe
3162 for_each_kmod_dep() {
3163     local func="${1:?}"
3164     local kmod="${2:?}"
3165     local found=0
3166     local cmd modpath
3167     shift 2
3168
3169     while read -r cmd modpath _; do
3170         [[ "$cmd" = insmod ]] || continue
3171         "$func" "$modpath" || return $?
3172         found=1
3173     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
3174
3175     ! get_bool "$found" && return 1
3176     return 0
3177 }
3178
3179 # instmods [-c] <kernel module> [<kernel module> ... ]
3180 # instmods [-c] <kernel subsystem>
3181 # install kernel modules along with all their dependencies.
3182 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
3183 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
3184 instmods() {
3185     local check=no
3186     if [[ $# -ge 0 && "$1" = '-c' ]]; then
3187         check=yes
3188         shift
3189     fi
3190
3191     inst1mod() {
3192         local mod="${1:?}"
3193         local ret=0
3194         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
3195
3196         case "$mod" in
3197             =*)
3198                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
3199                     (
3200                         [[ "$mpargs" ]] && echo "$mpargs"
3201                         cat "${mod_dir}/modules.${mod#=}"
3202                     ) | instmods
3203                 else
3204                     (
3205                         [[ "$mpargs" ]] && echo "$mpargs"
3206                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
3207                     ) | instmods
3208                 fi
3209                 ;;
3210             --*)
3211                 mpargs+=" $mod"
3212                 ;;
3213             i2o_scsi)
3214                 # Do not load this diagnostic-only module
3215                 return
3216                 ;;
3217             *)
3218                 mod=${mod##*/}
3219                 # if we are already installed, skip this module and go on
3220                 # to the next one.
3221                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
3222
3223                 # We use '-d' option in modprobe only if modules prefix path
3224                 # differs from default '/'.  This allows us to use Dracut with
3225                 # old version of modprobe which doesn't have '-d' option.
3226                 local mod_dirname=${mod_dir%%/lib/modules/*}
3227                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
3228
3229                 # ok, load the module, all its dependencies, and any firmware
3230                 # it may require
3231                 for_each_kmod_dep install_kmod_with_fw "$mod" \
3232                     --set-version "$KERNEL_VER" \
3233                     ${mod_dirname:+"$mod_dirname"} \
3234                     ${mpargs:+"$mpargs"}
3235                 ((ret+=$?))
3236                 ;;
3237         esac
3238         return "$ret"
3239     }
3240
3241     local mod mpargs
3242
3243     if [[ $# -eq 0 ]]; then  # filenames from stdin
3244         while read -r mod; do
3245             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3246                 dfatal "Failed to install $mod"
3247                 return 1
3248             fi
3249         done
3250     fi
3251
3252     for mod in "$@"; do # filenames as arguments
3253         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3254             dfatal "Failed to install $mod"
3255             return 1
3256         fi
3257     done
3258
3259     return 0
3260 }
3261
3262 _umount_dir() {
3263     local mountpoint="${1:?}"
3264     if mountpoint -q "$mountpoint"; then
3265         ddebug "umount $mountpoint"
3266         umount "$mountpoint"
3267     fi
3268 }
3269
3270 # can be overridden in specific test
3271 test_setup_cleanup() {
3272     cleanup_initdir
3273 }
3274
3275 _test_cleanup() {
3276     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3277     (
3278         set +e
3279         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3280         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3281         # If multiple setups/cleans are ran in parallel, this can cause a race
3282         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3283             rm -vf "${IMAGESTATEDIR}/default.img"
3284         fi
3285         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3286         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3287     ) || :
3288 }
3289
3290 # can be overridden in specific test
3291 test_cleanup() {
3292     _test_cleanup
3293 }
3294
3295 test_cleanup_again() {
3296     [ -n "$TESTDIR" ] || return
3297     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3298     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3299 }
3300
3301 test_create_image() {
3302     create_empty_image_rootdir
3303
3304     # Create what will eventually be our root filesystem onto an overlay
3305     (
3306         LOG_LEVEL=5
3307         setup_basic_environment
3308     )
3309 }
3310
3311 test_setup() {
3312     if ! get_bool "$NO_BUILD" && \
3313          get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3314          command -v meson >/dev/null && \
3315          [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3316         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3317         exit 1
3318     fi
3319
3320     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3321         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3322         mount_initdir
3323     else
3324         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3325             # default.img is the base that every test uses and optionally appends to
3326             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3327                 # Create the backing public image, but then completely unmount
3328                 # it and drop the loopback device responsible for it, since we're
3329                 # going to symlink/copy the image and mount it again from
3330                 # elsewhere.
3331                 local image_old="${IMAGE_PUBLIC}"
3332                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3333                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3334                 fi
3335                 test_create_image
3336                 test_setup_cleanup
3337                 umount_loopback
3338                 cleanup_loopdev
3339                 IMAGE_PUBLIC="${image_old}"
3340             fi
3341             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3342                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3343             fi
3344         fi
3345
3346         local hook_defined
3347         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3348
3349         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3350         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3351             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3352         else
3353             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3354         fi
3355
3356         mount_initdir
3357
3358         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3359             dinfo "Masking supporting services"
3360             mask_supporting_services
3361         fi
3362
3363         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
3364             # Do an initial coverage capture, to make sure the final report includes
3365             # files that the tests didn't touch at all
3366             lcov --initial --capture --directory "${initdir}/${BUILD_DIR:?}" --output-file "${TESTDIR:?}/coverage-base"
3367         fi
3368
3369         if get_bool "$hook_defined"; then
3370             test_append_files "${initdir:?}"
3371         fi
3372     fi
3373
3374     setup_nspawn_root
3375 }
3376
3377 test_run() {
3378     local test_id="${1:?}"
3379     mount_initdir
3380
3381     if ! get_bool "${TEST_NO_QEMU:=}"; then
3382         if run_qemu "$test_id"; then
3383             check_result_qemu || { echo "qemu test failed"; return 1; }
3384         else
3385             dwarn "can't run qemu, skipping"
3386         fi
3387     fi
3388     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3389         mount_initdir
3390         if run_nspawn "${initdir:?}" "$test_id"; then
3391             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3392         else
3393             dwarn "can't run systemd-nspawn, skipping"
3394         fi
3395
3396         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3397             dir="$TESTDIR/unprivileged-nspawn-root"
3398             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3399                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3400             else
3401                 dwarn "can't run systemd-nspawn, skipping"
3402             fi
3403         fi
3404     fi
3405     return 0
3406 }
3407
3408 do_test() {
3409     if [[ $UID != "0" ]]; then
3410         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3411         exit 0
3412     fi
3413
3414     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3415         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3416         exit 0
3417     fi
3418
3419     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3420         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3421         exit 0
3422     fi
3423
3424     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3425         TEST_NO_QEMU=1
3426     fi
3427
3428     # Detect lib paths
3429     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3430         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3431     done
3432
3433     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3434         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3435     done
3436
3437     mkdir -p "$WORKDIR"
3438     mkdir -p "$STATEDIR"
3439
3440     import_testdir
3441     import_initdir
3442
3443     if [ -n "${SUDO_USER}" ]; then
3444         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3445         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3446     fi
3447
3448     testname="$(basename "$PWD")"
3449
3450     while (($# > 0)); do
3451         case $1 in
3452             --run)
3453                 echo "${testname} RUN: $TEST_DESCRIPTION"
3454                 test_run "$TESTID"
3455                 ret=$?
3456                 if [ $ret -eq 0 ]; then
3457                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3458                 else
3459                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3460                 fi
3461                 exit $ret
3462                 ;;
3463             --setup)
3464                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3465                 test_setup
3466                 test_setup_cleanup
3467                 ;;
3468             --clean)
3469                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3470                 test_cleanup
3471                 ;;
3472             --clean-again)
3473                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3474                 test_cleanup_again
3475                 ;;
3476             --all)
3477                 ret=0
3478                 echo -n "${testname}: $TEST_DESCRIPTION "
3479                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3480                 # and loop devices will leak
3481                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3482                 if [ $ret -eq 0 ]; then
3483                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3484                 fi
3485                 if [ $ret -eq 0 ]; then
3486                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3487                 fi
3488                 test_cleanup
3489                 if [ $ret -eq 0 ]; then
3490                     # $TESTLOG is in $STATEDIR, so clean it up only on success
3491                     [[ -n "$STATEDIR" ]] && rm -vfr "$STATEDIR"
3492                     echo "[OK]"
3493                 else
3494                     echo "[FAILED]"
3495                     echo "see $TESTLOG"
3496                 fi
3497                 exit $ret
3498                 ;;
3499             *)
3500                 break
3501                 ;;
3502         esac
3503         shift
3504     done
3505 }