test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 # Simple wrapper to unify boolean checks.
  18 # Note: this function needs to stay near the top of the file, so we can use it
  19 #       in code in the outermost scope.
  20 get_bool() {
  21     # Make the value lowercase to make the regex matching simpler
  22     local _bool="${1,,}"
  23
  24     # Consider empty value as "false"
  25     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  26         return 1
  27     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  28         return 0
  29     else
  30         echo >&2 "Value '$_bool' is not a valid boolean value"
  31         exit 1
  32     fi
  33 }
  34
  35 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  36 export PATH
  37
  38 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  39 # shellcheck source=/dev/null
  40 source "$os_release"
  41 [[ "$ID" == "debian" || " $ID_LIKE " == *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=no
  42 [[ "$ID" == "arch" || " $ID_LIKE " == *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=no
  43 [[ "$ID" == "fedora" ]] && LOOKS_LIKE_FEDORA=yes || LOOKS_LIKE_FEDORA=no
  44 [[ " $ID_LIKE " == *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=no
  45
  46 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  47 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  48 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  49 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  50 get_bool "$LOOKS_LIKE_SUSE" && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  51 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  52 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  53 # Note that defining a different IMAGE_NAME in a test setup script will only result
  54 # in default.img being copied and renamed. It can then be extended by defining
  55 # a test_append_files() function. The $1 parameter will be the root directory.
  56 # To force creating a new image from scratch (eg: to encrypt it), also define
  57 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  58 IMAGE_NAME=${IMAGE_NAME:-default}
  59 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  60 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  61 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  62 LOOPDEV=
  63
  64 # Since in Bash we can have only one handler per signal, let's overcome this
  65 # limitation by having one global handler for the EXIT signal which executes
  66 # all registered handlers
  67 _AT_EXIT_HANDLERS=()
  68 _at_exit() {
  69     set +e
  70
  71     # Run the EXIT handlers in reverse order
  72     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  73         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  74         eval "${_AT_EXIT_HANDLERS[$i]}"
  75     done
  76 }
  77
  78 trap _at_exit EXIT
  79
  80 add_at_exit_handler() {
  81     _AT_EXIT_HANDLERS+=("${1:?}")
  82 }
  83
  84 # Decide if we can (and want to) run qemu with KVM acceleration.
  85 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  86 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  87 # running under KVM. If these conditions are met, enable KVM (and possibly
  88 # nested KVM), otherwise disable it.
  89 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  90     QEMU_KVM=yes
  91 else
  92     QEMU_KVM=no
  93 fi
  94
  95 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
  96     echo "WARNING! Cannot determine libdir from pkg-config, assuming /usr/lib/systemd" >&2
  97     ROOTLIBDIR=/usr/lib/systemd
  98 fi
  99
 100 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
 101 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 102 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 103 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 104 # These variables are used by test scripts
 105 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR
 106
 107 TOOLS_DIR="$SOURCE_DIR/tools"
 108 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 109 if get_bool "${NO_BUILD:=}"; then
 110     BUILD_DIR="$SOURCE_DIR"
 111 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 112     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 113     exit 1
 114 fi
 115
 116 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 117 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 118 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 119 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 120 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 121 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 122 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 123
 124 TESTFILE="${BASH_SOURCE[1]}"
 125 if [ -z "$TESTFILE" ]; then
 126     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 127     exit 1
 128 fi
 129 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 130
 131 WORKDIR="/var/tmp/systemd-tests"
 132 if get_bool "${NO_BUILD:=}"; then
 133     STATEDIR="$WORKDIR/$TESTNAME"
 134 else
 135     STATEDIR="$BUILD_DIR/test/$TESTNAME"
 136 fi
 137
 138 STATEFILE="$STATEDIR/.testdir"
 139 IMAGESTATEDIR="$STATEDIR/.."
 140 TESTLOG="$STATEDIR/test.log"
 141
 142 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 143     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 144     exit 1
 145 fi
 146 TESTID="${BASH_REMATCH[1]:?}"
 147
 148 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 149     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 150     exit 1
 151 fi
 152
 153 BASICTOOLS=(
 154     awk
 155     base64
 156     basename
 157     bash
 158     capsh
 159     cat
 160     chgrp
 161     chmod
 162     chown
 163     chroot
 164     cmp
 165     cp
 166     cryptsetup
 167     cut
 168     date
 169     dd
 170     dhclient
 171     diff
 172     dirname
 173     dmsetup
 174     echo
 175     env
 176     false
 177     find
 178     findmnt
 179     flock
 180     getconf
 181     getent
 182     getfacl
 183     getfattr
 184     grep
 185     grep
 186     gunzip
 187     gzip
 188     head
 189     hostname
 190     id
 191     ionice
 192     ip
 193     jq
 194     killall
 195     ldd
 196     ln
 197     ln
 198     loadkeys
 199     login
 200     losetup
 201     ls
 202     lsattr
 203     lsblk
 204     lz4cat
 205     mkdir
 206     mkfifo
 207     mknod
 208     mktemp
 209     modprobe
 210     mount
 211     mountpoint
 212     mv
 213     nc
 214     nproc
 215     ping
 216     pkill
 217     ps
 218     readlink
 219     realpath
 220     rev
 221     rm
 222     rmdir
 223     rmmod
 224     route
 225     script
 226     sed
 227     seq
 228     setfacl
 229     setfattr
 230     setfont
 231     setpriv
 232     setsid
 233     sfdisk
 234     sh
 235     sleep
 236     sort
 237     stat
 238     stty
 239     su
 240     sulogin
 241     sysctl
 242     tail
 243     tar
 244     tee
 245     test
 246     timeout
 247     touch
 248     tr
 249     true
 250     truncate
 251     tty
 252     umount
 253     uname
 254     unshare
 255     useradd
 256     userdel
 257     wc
 258     xargs
 259     xzcat
 260 )
 261
 262 DEBUGTOOLS=(
 263     df
 264     dmesg
 265     du
 266     free
 267     less
 268     strace
 269     vi
 270     /usr/libexec/vi
 271 )
 272
 273 is_built_with_asan() {
 274     local _bin="${1:?}"
 275
 276     if ! type -P objdump >/dev/null; then
 277         echo "Failed to find objdump, assuming systemd hasn't been built with ASAN."
 278         return 1
 279     fi
 280
 281     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 282     local _asan_calls
 283     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 284     if ((_asan_calls < 1000)); then
 285         return 1
 286     else
 287         return 0
 288     fi
 289 }
 290
 291 is_built_with_coverage() {
 292     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 293         return 1
 294     fi
 295
 296     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 297 }
 298
 299 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 300 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 301
 302 if get_bool "$IS_BUILT_WITH_ASAN"; then
 303     PATH_TO_INIT="$ROOTLIBDIR/systemd-under-asan"
 304     QEMU_MEM="${QEMU_MEM:-2G}"
 305     QEMU_SMP="${QEMU_SMP:-4}"
 306
 307     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 308     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 309         ASAN_COMPILER=gcc
 310         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 311     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 312         ASAN_COMPILER=clang
 313         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 314
 315         # As clang's ASan DSO is usually in a non-standard path, let's check if
 316         # the environment is set accordingly. If not, warn the user and exit.
 317         # We're not setting the LD_LIBRARY_PATH automagically here, because
 318         # user should encounter (and fix) the same issue when running the unit
 319         # tests (meson test)
 320         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 321             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 322             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 323             exit 1
 324         fi
 325     else
 326         echo >&2 "systemd is not linked against the ASan DSO"
 327         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 328         exit 1
 329     fi
 330
 331     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 332 fi
 333
 334 test_require_bin() {
 335     local bin
 336
 337     for bin in "$@"; do
 338         if ! command -v "$bin" >/dev/null; then
 339             echo "Required binary $bin not available, skipping the test"
 340             exit 0
 341         fi
 342     done
 343 }
 344
 345 find_qemu_bin() {
 346     QEMU_BIN="${QEMU_BIN:-""}"
 347     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 348     if get_bool "$QEMU_KVM"; then
 349         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 350     fi
 351
 352     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 353     case $ARCH in
 354     x86_64)
 355         # QEMU's own build system calls it qemu-system-x86_64
 356         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 357         ;;
 358     i*86)
 359         # new i386 version of QEMU
 360         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 361
 362         # i386 version of QEMU
 363         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 364         ;;
 365     ppc64*)
 366         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 367         ;;
 368     esac
 369
 370     if [[ ! -e "$QEMU_BIN" ]]; then
 371         echo "Could not find a suitable qemu binary" >&2
 372         return 1
 373     fi
 374 }
 375
 376 qemu_setup_swtpm_socket() {
 377     local pid state_dir tpm_device
 378
 379     if ! tpm_device="$(qemu_get_tpm_device)"; then
 380         dinfo "Found QEMU version is too old for TPM2 on ppc64le"
 381         exit 0
 382     fi
 383
 384     state_dir="$(mktemp -d)"
 385     swtpm socket --tpm2 --tpmstate dir="$state_dir" --ctrl type=unixio,path="$state_dir/sock" &
 386     pid=$!
 387     if ! kill -0 "$pid"; then
 388         derror "Failed to start swtpm"
 389         return 1
 390     fi
 391
 392     if ! timeout 5 bash -c "until [[ -S $state_dir/sock ]]; do sleep .5; done"; then
 393         derror "Failed to setup swtpm socket"
 394         return 1
 395     fi
 396
 397     dinfo "Started swtpm as PID $pid with state dir $state_dir"
 398
 399     add_at_exit_handler "kill -TERM $pid 2>/dev/null; rm -rf '$state_dir'"
 400
 401     QEMU_OPTIONS+=" -chardev socket,id=chrtpm,path=$state_dir/sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device $tpm_device,tpmdev=tpm0"
 402     dinfo "Configured emulated TPM2 device $tpm_device"
 403
 404     return 0
 405 }
 406
 407 qemu_get_tpm_device() {
 408     local tpm_device="tpm-tis"
 409
 410     if [[ "$(uname -m)" == "ppc64le" ]]; then
 411         # tpm-spapr support was introduced in qemu 5.0.0
 412         if ! qemu_min_version "5.0.0"; then
 413             return 1
 414         fi
 415
 416         tpm_device="tpm-spapr"
 417     fi
 418
 419     echo "$tpm_device"
 420     return 0
 421 }
 422
 423 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 424 # returns 0 if newer or equal
 425 # returns 1 if older
 426 # returns 2 if failing
 427 qemu_min_version() {
 428     find_qemu_bin || return 2
 429
 430     # get version from binary
 431     local qemu_ver
 432     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 433
 434     # Check version string format
 435     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 436     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 437
 438     # compare as last command to return that value
 439     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 440 }
 441
 442 # Pads a file to multiple of 4 bytes
 443 pad4_file() {
 444     local size
 445     size=$(stat -c "%s" "$1")
 446     local padded
 447     padded=$((((size + 3) / 4) * 4))
 448     truncate -s "$padded" "$1"
 449 }
 450
 451 # Return 0 if qemu did run (then you must check the result state/logs for actual
 452 # success), or 1 if qemu is not available.
 453 run_qemu() {
 454     if declare -F run_qemu_hook >/dev/null; then
 455         if ! run_qemu_hook "${workspace}"; then
 456             derror "check_qemu_hook() returned with EC > 0"
 457             ret=4
 458         fi
 459     fi
 460
 461     # If the test provided its own initrd, use it (e.g. TEST-24)
 462     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 463         INITRD="$TESTDIR/initrd.img"
 464     fi
 465
 466     if [ -f /etc/machine-id ]; then
 467         read -r MACHINE_ID </etc/machine-id
 468         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 469             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 470         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 471             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 472     fi
 473
 474     local CONSOLE=ttyS0
 475
 476     # Reset the boot counter, if present
 477     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 478     rm -f "$initdir"/{testok,failed,skipped}
 479     # make sure the initdir is not mounted to avoid concurrent access
 480     cleanup_initdir
 481     umount_loopback
 482
 483     if [[ ! "$KERNEL_BIN" ]]; then
 484         if get_bool "$LOOKS_LIKE_ARCH"; then
 485             KERNEL_BIN=/boot/vmlinuz-linux
 486         else
 487             [ "$ARCH" ] || ARCH=$(uname -m)
 488             case $ARCH in
 489                 ppc64*)
 490                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 491                     # uses the "standard" vmlinuz- prefix
 492                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 493                     CONSOLE=hvc0
 494                     ;;
 495                 *)
 496                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 497                     ;;
 498             esac
 499         fi
 500     fi
 501
 502     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 503     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 504     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 505     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 506     if [[ ! "$INITRD" ]]; then
 507         if [[ -e "$default_fedora_initrd" ]]; then
 508             INITRD="$default_fedora_initrd"
 509         elif get_bool "$LOOKS_LIKE_DEBIAN" && [[ -e "$default_debian_initrd" ]]; then
 510             INITRD="$default_debian_initrd"
 511         elif get_bool "$LOOKS_LIKE_ARCH" && [[ -e "$default_arch_initrd" ]]; then
 512             INITRD="$default_arch_initrd"
 513         elif get_bool "$LOOKS_LIKE_SUSE" && [[ -e "$default_suse_initrd" ]]; then
 514             INITRD="$default_suse_initrd"
 515         fi
 516     fi
 517
 518     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 519     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 520     # is not installed or there's some other error when calling it, fall back
 521     # to the original value (QEMU_SMP=1).
 522     if [[ -z "${QEMU_SMP:=}" ]]; then
 523         if ! QEMU_SMP=$(nproc); then
 524             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 525             QEMU_SMP=1
 526         fi
 527     fi
 528
 529     find_qemu_bin || return 1
 530
 531     if get_bool "${TEST_SETUP_SWTPM:-}"; then
 532         qemu_setup_swtpm_socket || return 1
 533     fi
 534
 535     # Umount initdir to avoid concurrent access to the filesystem
 536     _umount_dir "$initdir"
 537
 538     local kernel_params=()
 539     local qemu_options=()
 540     local qemu_cmd=("$QEMU_BIN")
 541
 542     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 543         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 544     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 545         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 546     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 547         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 548     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 549         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 550         exit 1
 551     fi
 552
 553     if get_bool "$LOOKS_LIKE_SUSE"; then
 554         kernel_params+=("rd.hostonly=0")
 555     fi
 556
 557     # Debian/Ubuntu's initramfs tries to check if it can resume from hibernation
 558     # and wastes a minute or so probing disks, skip that as it's not useful here
 559     kernel_params+=(
 560         "root=LABEL=systemd_boot"
 561         "rw"
 562         "raid=noautodetect"
 563         "rd.luks=0"
 564         "loglevel=2"
 565         "init=$PATH_TO_INIT"
 566         "console=$CONSOLE"
 567         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 568         "systemd.unit=testsuite.target"
 569         "systemd.wants=testsuite-$1.service"
 570         "noresume"
 571         "oops=panic"
 572         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 573         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 574     )
 575
 576     if ! get_bool "$INTERACTIVE_DEBUG" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 577         kernel_params+=(
 578             "panic=1"
 579             "softlockup_panic=1"
 580             "systemd.wants=end.service"
 581         )
 582     fi
 583
 584     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 585     qemu_options+=(
 586         -smp "$QEMU_SMP"
 587         -net none
 588         -m "${QEMU_MEM:-768M}"
 589         -nographic
 590         -kernel "$KERNEL_BIN"
 591         -drive "format=raw,cache=unsafe,file=$image"
 592         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 593     )
 594
 595     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 596         local user_qemu_options
 597         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 598         qemu_options+=("${user_qemu_options[@]}")
 599     fi
 600     qemu_options+=(${QEMU_OPTIONS_ARRAY:+"${QEMU_OPTIONS_ARRAY[@]}"})
 601
 602     if [[ -n "$INITRD" ]]; then
 603         if [[ -n "$INITRD_EXTRA" ]]; then
 604             # An addition initrd has been specified, let's combine it with the main one.
 605             local t="$WORKDIR"/initrd.combined."$RANDOM"
 606
 607             # First, show contents of additional initrd
 608             echo "Additional initrd contents:"
 609             cpio -tv < "$INITRD_EXTRA"
 610
 611             # Copy the main initrd
 612             zstd -d -c -f "$INITRD" > "$t"
 613             add_at_exit_handler "rm $t"
 614             # Kernel requires this to be padded to multiple of 4 bytes with zeroes
 615             pad4_file "$t"
 616
 617             # Copy the additional initrd
 618             cat "$INITRD_EXTRA" >> "$t"
 619             pad4_file "$t"
 620
 621             qemu_options+=(-initrd "$t")
 622         else
 623             qemu_options+=(-initrd "$INITRD")
 624         fi
 625     fi
 626
 627     # Let's use KVM if possible
 628     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 629         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 630     fi
 631
 632     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 633         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 634     fi
 635
 636     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]} ${KERNEL_APPEND:-}" |& tee "${TESTDIR:?}/console.log")
 637     rc=$?
 638     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 639         derror "Test timed out after ${QEMU_TIMEOUT}s"
 640         TIMED_OUT=1
 641     else
 642         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 643     fi
 644     return 0
 645 }
 646
 647 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 648 # success), or 1 if nspawn is not available.
 649 run_nspawn() {
 650     [[ -d /run/systemd/system ]] || return 1
 651     # Reset the boot counter, if present
 652     rm -f "${initdir:?}/var/tmp/.systemd_reboot_count"
 653     rm -f "${initdir:?}"/{testok,failed,skipped}
 654
 655     local nspawn_cmd=()
 656     local nspawn_options=(
 657         "--register=no"
 658         "--kill-signal=SIGKILL"
 659         "--directory=${1:?}"
 660         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 661         "--machine=TEST-$TESTID"
 662     )
 663     local kernel_params=(
 664         "$PATH_TO_INIT"
 665         "systemd.unit=testsuite.target"
 666         "systemd.wants=testsuite-$2.service"
 667         ${TEST_MATCH_SUBTEST:+"systemd.setenv=TEST_MATCH_SUBTEST=$TEST_MATCH_SUBTEST"}
 668         ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
 669     )
 670
 671     if get_bool "$INTERACTIVE_DEBUG"; then
 672         nspawn_options+=("--console=interactive")
 673     elif ! get_bool "$TEST_SKIP_SHUTDOWN"; then
 674         kernel_params+=("systemd.wants=end.service")
 675     fi
 676
 677     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 678         local user_nspawn_arguments
 679         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 680         nspawn_options+=("${user_nspawn_arguments[@]}")
 681     fi
 682
 683     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 684         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 685         exit
 686     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 687         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 688     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 689         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 690     else
 691         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 692         exit 1
 693     fi
 694
 695     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 696         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 697     else
 698         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 699     fi
 700
 701     # Word splitting here is intentional
 702     # shellcheck disable=SC2086
 703     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}" ${KERNEL_APPEND:-} |& tee "${TESTDIR:?}/console.log")
 704     rc=$?
 705     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 706         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 707         TIMED_OUT=1
 708     else
 709         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 710     fi
 711     return 0
 712 }
 713
 714 # Build two very minimal root images, with two units, one is the same and one is different across them
 715 install_verity_minimal() {
 716     dinfo "Set up a set of minimal images for verity verification"
 717     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 718         return
 719     fi
 720     if ! command -v mksquashfs >/dev/null 2>&1; then
 721         dfatal "mksquashfs not found"
 722         exit 1
 723     fi
 724     if ! command -v veritysetup >/dev/null 2>&1; then
 725         dfatal "veritysetup not found"
 726         exit 1
 727     fi
 728     # Local modifications of some global variables is intentional in this
 729     # subshell (SC2030)
 730     # shellcheck disable=SC2030
 731     (
 732         BASICTOOLS=(
 733             bash
 734             cat
 735             grep
 736             mount
 737             sleep
 738             touch
 739         )
 740         oldinitdir="$initdir"
 741         rm -rfv "$TESTDIR/minimal"
 742         export initdir="$TESTDIR/minimal"
 743         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 744         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 745         setup_basic_dirs
 746         install_basic_tools
 747         install_ld_so_conf
 748         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 749         # issue, thus falsely triggering SC2030 in this case
 750         # See: koalaman/shellcheck#1409
 751         if [[ -v ASAN_RT_PATH ]]; then
 752             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 753             # into the verity images to get rid of the annoying errors about
 754             # missing $LD_PRELOAD libraries.
 755             inst_libs "$ASAN_RT_PATH"
 756             inst_library "$ASAN_RT_PATH"
 757         fi
 758         cp "$os_release" "$initdir/usr/lib/os-release"
 759         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 760         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 761         touch "$initdir/opt/some_file"
 762         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 763         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 764         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 765 [Service]
 766 ExecStartPre=cat /usr/lib/os-release
 767 ExecStart=sleep 120
 768 EOF
 769         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 770
 771         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 772         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 773             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 774
 775         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 776         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 777         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 778
 779         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 780         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 781             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 782
 783         # Rolling distros like Arch do not set VERSION_ID
 784         local version_id=""
 785         if grep -q "^VERSION_ID=" "$os_release"; then
 786             version_id="$(grep "^VERSION_ID=" "$os_release")"
 787         fi
 788
 789         export initdir="$TESTDIR/app0"
 790         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 791         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 792         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 793         ( echo "${version_id}"
 794           echo "SYSEXT_IMAGE_ID=app" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 795         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 796 [Service]
 797 Type=oneshot
 798 RemainAfterExit=yes
 799 ExecStart=/opt/script0.sh
 800 TemporaryFileSystem=/var/lib
 801 StateDirectory=app0
 802 RuntimeDirectory=app0
 803 EOF
 804         cat >"$initdir/opt/script0.sh" <<EOF
 805 #!/bin/bash
 806 set -e
 807 test -e /usr/lib/os-release
 808 echo bar >\${STATE_DIRECTORY}/foo
 809 cat /usr/lib/extension-release.d/extension-release.app0
 810 EOF
 811         chmod +x "$initdir/opt/script0.sh"
 812         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 813         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 814
 815         export initdir="$TESTDIR/conf0"
 816         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system" "$initdir/opt"
 817         grep "^ID=" "$os_release" >"$initdir/etc/extension-release.d/extension-release.conf0"
 818         echo "${version_id}" >>"$initdir/etc/extension-release.d/extension-release.conf0"
 819         ( echo "${version_id}"
 820           echo "CONFEXT_IMAGE_ID=app" ) >>"$initdir/etc/extension-release.d/extension-release.conf0"
 821         echo MARKER_1 >"$initdir/etc/systemd/system/some_file"
 822         mksquashfs "$initdir" "$oldinitdir/usr/share/conf0.raw" -noappend
 823
 824         export initdir="$TESTDIR/app1"
 825         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 826         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 827         ( echo "${version_id}"
 828           echo "SYSEXT_SCOPE=portable"
 829           echo "SYSEXT_IMAGE_ID=app"
 830           echo "SYSEXT_IMAGE_VERSION=1"
 831           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 832         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 833         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 834 [Service]
 835 Type=oneshot
 836 RemainAfterExit=yes
 837 ExecStart=/opt/script1.sh
 838 StateDirectory=app1
 839 RuntimeDirectory=app1
 840 EOF
 841         cat >"$initdir/opt/script1.sh" <<EOF
 842 #!/bin/bash
 843 set -e
 844 test -e /usr/lib/os-release
 845 echo baz >\${STATE_DIRECTORY}/foo
 846 cat /usr/lib/extension-release.d/extension-release.app2
 847 EOF
 848         chmod +x "$initdir/opt/script1.sh"
 849         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 850         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 851
 852         export initdir="$TESTDIR/app-nodistro"
 853         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 854         ( echo "ID=_any"
 855           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 856         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 857         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 858
 859         export initdir="$TESTDIR/service-scoped-test"
 860         mkdir -p "$initdir/etc/extension-release.d" "$initdir/etc/systemd/system"
 861         ( echo "ID=_any"
 862           echo "ARCHITECTURE=_any" ) >"$initdir/etc/extension-release.d/extension-release.service-scoped-test"
 863         echo MARKER_CONFEXT_123 >"$initdir/etc/systemd/system/some_file"
 864         mksquashfs "$initdir" "$oldinitdir/etc/service-scoped-test.raw" -noappend
 865
 866         # We need to create a dedicated sysext image to test the reload mechanism. If we share an image to install the
 867         # 'foo.service' it will be loaded from another test run, which will impact the targeted test.
 868         export initdir="$TESTDIR/app-reload"
 869         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 870         ( echo "ID=_any"
 871           echo "ARCHITECTURE=_any"
 872           echo "EXTENSION_RELOAD_MANAGER=1" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-reload"
 873         mkdir -p "$initdir/usr/lib/systemd/system/multi-user.target.d"
 874         cat >"${initdir}/usr/lib/systemd/system/foo.service" <<EOF
 875 [Service]
 876 Type=oneshot
 877 RemainAfterExit=yes
 878 ExecStart=echo foo
 879
 880 [Install]
 881 WantedBy=multi-user.target
 882 EOF
 883         { echo "[Unit]"; echo "Upholds=foo.service"; } > "$initdir/usr/lib/systemd/system/multi-user.target.d/10-foo-service.conf"
 884         mksquashfs "$initdir" "$oldinitdir/usr/share/app-reload.raw" -noappend
 885     )
 886 }
 887
 888 setup_basic_environment() {
 889     # create the basic filesystem layout
 890     setup_basic_dirs
 891
 892     install_systemd
 893     install_missing_libraries
 894     install_config_files
 895     install_zoneinfo
 896     create_rc_local
 897     install_basic_tools
 898     install_libnss
 899     install_pam
 900     install_dbus
 901     install_fonts
 902     install_locales
 903     install_keymaps
 904     install_x11_keymaps
 905     install_terminfo
 906     install_execs
 907     install_fs_tools
 908     install_modules
 909     install_plymouth
 910     install_haveged
 911     install_debug_tools
 912     install_ld_so_conf
 913     install_testuser
 914     has_user_dbus_socket && install_user_dbus
 915     setup_selinux
 916     install_depmod_files
 917     generate_module_dependencies
 918     if get_bool "$IS_BUILT_WITH_ASAN"; then
 919         create_asan_wrapper
 920     fi
 921     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 922         install_verity_minimal
 923     fi
 924 }
 925
 926 setup_selinux() {
 927     dinfo "Setup SELinux"
 928     # don't forget KERNEL_APPEND='... selinux=1 ...'
 929     if ! get_bool "$SETUP_SELINUX"; then
 930         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 931         return 0
 932     fi
 933
 934     for dir in /etc/selinux /usr/share/selinux; do
 935         rm -rf "${initdir:?}/$dir"
 936         if ! cp -ar "$dir" "$initdir/$dir"; then
 937             dfatal "Failed to copy $dir"
 938             exit 1
 939         fi
 940     done
 941
 942     # We use a custom autorelabel service instead of the SELinux provided set
 943     # of units & a generator, since the generator overrides the default target
 944     # to the SELinux one when it detects /.autorelabel. However, we use
 945     # systemd.unit= on the kernel command cmdline which always takes precedence,
 946     # rendering all SELinux efforts useless. Also, pulling in selinux-autorelabel.service
 947     # explicitly doesn't work either, as it doesn't check for the presence of /.autorelabel
 948     # and does the relabeling unconditionally which always ends with a reboot, so
 949     # we end up in a reboot loop (and it also spews quite a lot of errors as it
 950     # wants /etc/fstab and dracut-initramfs-restore).
 951     touch "$initdir/.autorelabel"
 952     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 953     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 954
 955     # Tools requires by fixfiles
 956     image_install awk bash cat chcon expr egrep find grep head secon setfiles rm sort uname uniq
 957     image_install fixfiles getenforce load_policy selinuxenabled sestatus
 958 }
 959
 960 install_valgrind() {
 961     if ! type -p valgrind; then
 962         dfatal "Failed to install valgrind"
 963         exit 1
 964     fi
 965
 966     local valgrind_bins valgrind_libs valgrind_supp
 967
 968     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null |
 969                                        sed -r -n 's/execve\("([^"]*)".*/\1/p')
 970     image_install "${valgrind_bins[@]}"
 971
 972     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null |
 973                                        sed -r -n 's|.*calling init: (/.*vgpreload_.*)|\1|p')
 974     image_install "${valgrind_libs[@]}"
 975
 976     readarray -t valgrind_supp < <(strace -e open valgrind /bin/true 2>&1 >/dev/null |
 977                                        sed -r -n 's,open\("([^"]*(/debug[^"]*|\.supp))".*= [0-9].*,\1,p')
 978     image_install "${valgrind_supp[@]}"
 979 }
 980
 981 create_valgrind_wrapper() {
 982     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 983     ddebug "Create $valgrind_wrapper"
 984     cat >"$valgrind_wrapper" <<EOF
 985 #!/usr/bin/env bash
 986
 987 mount -t proc proc /proc
 988 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 989 EOF
 990     chmod 0755 "$valgrind_wrapper"
 991 }
 992
 993 create_asan_wrapper() {
 994     local asan_wrapper default_asan_options default_ubsan_options default_environment manager_environment
 995
 996     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 997
 998     default_asan_options="${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}"
 999     default_ubsan_options="${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}"
1000
1001     if [[ "$ASAN_COMPILER" == "clang" ]]; then
1002         # clang: install llvm-symbolizer to generate useful reports
1003         # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
1004         image_install "llvm-symbolizer"
1005
1006         # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
1007         # unnecessary for gcc & libasan, however, for clang this is crucial, as its
1008         # runtime ASan DSO is in a non-standard (library) path.
1009         mkdir -p "${initdir:?}/etc/ld.so.conf.d/"
1010         echo "${ASAN_RT_PATH%/*}" >"${initdir:?}/etc/ld.so.conf.d/asan-path-override.conf"
1011         ldconfig -r "$initdir"
1012     fi
1013
1014     # Create a simple environment file which can be included by systemd services
1015     # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
1016     cat >"${initdir:?}/usr/lib/systemd/systemd-asan-env" <<EOF
1017 LD_PRELOAD=$ASAN_RT_PATH
1018 ASAN_OPTIONS=$default_asan_options
1019 LSAN_OPTIONS=detect_leaks=0
1020 UBSAN_OPTIONS=$default_ubsan_options
1021 EOF
1022
1023     default_environment=(
1024         "ASAN_OPTIONS='$default_asan_options'"
1025         "UBSAN_OPTIONS='$default_ubsan_options'"
1026         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1027     )
1028     manager_environment=(
1029         "ASAN_OPTIONS='$default_asan_options:log_path=/systemd-pid1.asan.log:log_to_syslog=1'"
1030         "UBSAN_OPTIONS='$default_ubsan_options:log_path=/systemd-pid1.ubsan.log:log_to_syslog=1'"
1031         "ASAN_RT_PATH='$ASAN_RT_PATH'"
1032     )
1033
1034     mkdir -p "${initdir:?}/etc/systemd/system.conf.d/"
1035     cat >"${initdir:?}/etc/systemd/system.conf.d/asan.conf" <<EOF
1036 [Manager]
1037 DefaultEnvironment=${default_environment[*]}
1038 ManagerEnvironment=${manager_environment[*]}
1039 DefaultTimeoutStartSec=180s
1040 EOF
1041
1042     # ASAN and syscall filters aren't compatible with each other.
1043     find "${initdir:?}" -name '*.service' -type f -print0 | xargs -0 sed -i 's/^\(MemoryDeny\|SystemCall\)/#\1/'
1044
1045     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journald.service.d/"
1046     cat >"${initdir:?}/etc/systemd/system/systemd-journald.service.d/asan-env.conf" <<EOF
1047 [Service]
1048 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
1049 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
1050 Environment=ASAN_OPTIONS=$default_asan_options:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=$default_ubsan_options:log_path=/systemd-journald.ubsan.log
1051
1052 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
1053 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
1054 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
1055 StandardOutput=file:/systemd-journald.out
1056 EOF
1057
1058     # 90s isn't enough for some services to finish when literally everything is run
1059     # under ASan+UBSan in containers, which, in turn, are run in VMs.
1060     # Let's limit which environments such services should be executed in.
1061     mkdir -p "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/"
1062     cat >"${initdir:?}/etc/systemd/system/systemd-hwdb-update.service.d/asan.conf" <<EOF
1063 [Unit]
1064 ConditionVirtualization=container
1065
1066 [Service]
1067 TimeoutSec=240s
1068 EOF
1069
1070     # Let's override another hard-coded timeout that kicks in too early
1071     mkdir -p "${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/"
1072     cat >"${initdir:?}/etc/systemd/system/systemd-journal-flush.service.d/asan.conf" <<EOF
1073 [Service]
1074 TimeoutSec=180s
1075 EOF
1076
1077     asan_wrapper="${initdir:?}/${PATH_TO_INIT:?}"
1078     # Sanity check to make sure we don't overwrite something we shouldn't.
1079     [[ "$asan_wrapper" =~ systemd-under-asan$ ]]
1080
1081     cat >"$asan_wrapper" <<EOF
1082 #!/usr/bin/env bash
1083 set -eux
1084
1085 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
1086 export ${manager_environment[@]}
1087 [[ -n "\$ASAN_OPTIONS" && -n "\$UBSAN_OPTIONS" ]]
1088
1089 exec "$ROOTLIBDIR/systemd" "\$@"
1090 EOF
1091     chmod 0755 "$asan_wrapper"
1092 }
1093
1094 create_strace_wrapper() {
1095     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
1096     ddebug "Create $strace_wrapper"
1097     cat >"$strace_wrapper" <<EOF
1098 #!/usr/bin/env bash
1099
1100 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
1101 EOF
1102     chmod 0755 "$strace_wrapper"
1103 }
1104
1105 install_fs_tools() {
1106     dinfo "Install fsck"
1107     image_install /sbin/fsck*
1108     image_install -o /bin/fsck*
1109
1110     # fskc.reiserfs calls reiserfsck. so, install it
1111     image_install -o reiserfsck
1112
1113     # we use mkfs in system-repart tests
1114     image_install /sbin/mkfs.ext4
1115     image_install /sbin/mkfs.vfat
1116 }
1117
1118 install_modules() {
1119     dinfo "Install modules"
1120
1121     instmods bridge dummy ipvlan macvlan vfat veth
1122     instmods loop =block
1123     instmods nls_ascii =nls
1124     instmods overlay =overlayfs
1125     instmods scsi_debug
1126
1127     if get_bool "$LOOKS_LIKE_SUSE"; then
1128         instmods ext4 af_packet
1129     fi
1130 }
1131
1132 install_dmevent() {
1133     instmods dm_crypt =crypto
1134     inst_binary dmeventd
1135     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
1136     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1137         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
1138         # and since buster/bionic 95-dm-notify.rules
1139         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
1140         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
1141     else
1142         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
1143     fi
1144     if get_bool "$LOOKS_LIKE_SUSE"; then
1145         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
1146     fi
1147 }
1148
1149 install_multipath() {
1150     instmods "=md" multipath
1151     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
1152     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
1153     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1154         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
1155     else
1156         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
1157     fi
1158     mkdir -p "${initdir:?}/etc/multipath"
1159
1160     local file
1161     while read -r file; do
1162         # Install libraries required by the given library
1163         inst_libs "$file"
1164         # Install the library itself and create necessary symlinks
1165         inst_library "$file"
1166     done < <(find /lib*/multipath -type f)
1167 }
1168
1169 install_lvm() {
1170     local lvm_rules rule_prefix
1171
1172     image_install lvm
1173     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1174     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1175     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1176
1177     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1178         lvm_rules="56-lvm.rules"
1179         rule_prefix=""
1180     else
1181         lvm_rules="11-dm-lvm.rules"
1182         rule_prefix="dm-"
1183     fi
1184
1185     # Support the new udev autoactivation introduced in lvm 2.03.14
1186     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1187     # Static autoactivation (via lvm2-activation-generator) was dropped
1188     # in lvm 2.03.15
1189     # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1190     if [[ -f "/lib/udev/rules.d/69-${rule_prefix}lvm.rules" ]]; then
1191         inst_rules "$lvm_rules" "69-${rule_prefix}lvm.rules"
1192     else
1193         image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1194         image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1195         inst_rules "$lvm_rules" "69-${rule_prefix}lvm-metad.rules"
1196     fi
1197
1198     mkdir -p "${initdir:?}/etc/lvm"
1199 }
1200
1201 host_has_btrfs() (
1202     set -e
1203     modprobe -nv btrfs && command -v mkfs.btrfs && command -v btrfs || return $?
1204 )
1205
1206 install_btrfs() {
1207     instmods btrfs
1208     # Not all utilities provided by btrfs-progs are listed here; extend the list
1209     # if necessary
1210     image_install btrfs btrfstune mkfs.btrfs
1211     inst_rules 64-btrfs-dm.rules
1212 }
1213
1214 install_iscsi() {
1215     # Install both client and server side stuff by default
1216     local inst="${1:-}"
1217     local file
1218
1219     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1220     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1221     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1222         image_install iscsi-iname iscsiadm iscsid iscsistart
1223         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1224         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1225         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1226         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1227         mkdir -p "${initdir:?}/etc/iscsi"
1228         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1229         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1230         # a one-time service instead of distro package's post-install scripts.
1231         # However, some distros still use this approach even after this patch,
1232         # so prefer the already existing initiatorname.iscsi file if it exists.
1233         #
1234         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1235         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1236             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1237             if get_bool "$IS_BUILT_WITH_ASAN"; then
1238                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1239                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1240                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1241                 # (among others) pre-loads ASan's DSO.
1242                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1243                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1244             fi
1245         else
1246             inst_simple "/etc/iscsi/initiatorname.iscsi"
1247         fi
1248     fi
1249
1250     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1251     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1252     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1253         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1254         image_install -o /etc/sysconfig/tgtd
1255         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1256         mkdir -p "${initdir:?}/etc/tgt"
1257         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1258         # Install perl modules required by tgt-admin
1259         #
1260         # Forgive me father for I have sinned. The monstrosity below appends
1261         # a perl snippet to the `tgt-admin` perl script on the fly, which
1262         # dumps a list of files (perl modules) required by `tgt-admin` at
1263         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1264         # passed to `inst_simple` which installs the necessary files into the image
1265         #
1266         # shellcheck disable=SC2016
1267         while read -r file; do
1268             inst_simple "$file"
1269         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1270     fi
1271 }
1272
1273 host_has_mdadm() (
1274     set -e
1275     command -v mdadm || return $?
1276 )
1277
1278 install_mdadm() {
1279     local unit
1280     local mdadm_units=(
1281         system/mdadm-grow-continue@.service
1282         system/mdadm-last-resort@.service
1283         system/mdadm-last-resort@.timer
1284         system/mdmon@.service
1285         system/mdmonitor-oneshot.service
1286         system/mdmonitor-oneshot.timer
1287         system/mdmonitor.service
1288         system-shutdown/mdadm.shutdown
1289     )
1290
1291     instmods "=md"
1292     image_install mdadm mdmon
1293     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1294     # Fedora/CentOS/RHEL ships this rule file
1295     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1296
1297     for unit in "${mdadm_units[@]}"; do
1298         image_install "${ROOTLIBDIR:?}/$unit"
1299     done
1300
1301     # Disable the mdmonitor service, since it fails if there's no valid email address
1302     # configured in /etc/mdadm.conf, which just unnecessarily pollutes the logs
1303     "${SYSTEMCTL:?}" mask --root "${initdir:?}" mdmonitor.service || :
1304 }
1305
1306 install_compiled_systemd() {
1307     dinfo "Install compiled systemd"
1308
1309     local ninja_bin
1310     ninja_bin="$(type -P ninja || type -P ninja-build)"
1311     if [[ -z "$ninja_bin" ]]; then
1312         dfatal "ninja was not found"
1313         exit 1
1314     fi
1315     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1316
1317     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1318     # find them in the same directory as the runtime data counts
1319     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1320         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1321         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1322         # Set effective & default ACLs for the build dir so unprivileged
1323         # processes can write gcda files with coverage stats
1324         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1325     fi
1326 }
1327
1328 install_package_file() {
1329     local file="${1:?}"
1330
1331     # Skip missing files (like /etc/machine-info)
1332     [[ ! -e "$file" ]] && return 0
1333     # Skip python unit tests, since the image_install machinery will try to pull
1334     # in the whole python stack in a very questionable state, making the tests fail.
1335     # And given we're trying to transition to mkosi-based images anyway I'm not even
1336     # going to bother
1337     [[ "$file" =~ /tests/unit-tests/.*.py$ ]] && return 0
1338     # If the current file is a directory, create it with the original
1339     # mode; if it's a symlink to a directory, copy it as-is
1340     if [[ -d "$file" ]]; then
1341         inst_dir "$file"
1342     else
1343         inst "$file"
1344     fi
1345 }
1346
1347 install_debian_systemd() {
1348     dinfo "Install debian systemd"
1349
1350     local deb file
1351
1352     while read -r deb; do
1353         ddebug "Install debian files from package $deb"
1354         while read -r file; do
1355             install_package_file "$file"
1356         done < <(dpkg-query -L "$deb" 2>/dev/null)
1357     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1358 }
1359
1360 install_rpm() {
1361     local rpm="${1:?}"
1362     local file
1363
1364     if ! rpm -q "$rpm" >/dev/null; then
1365         derror "RPM $rpm is not installed"
1366         return 1
1367     fi
1368
1369     dinfo "Installing contents of RPM $rpm"
1370     while read -r file; do
1371         install_package_file "$file"
1372     done < <(rpm -ql "$rpm")
1373 }
1374
1375 install_suse_systemd() {
1376     local pkgs
1377
1378     dinfo "Install SUSE systemd"
1379
1380     pkgs=(
1381         systemd
1382         systemd-container
1383         systemd-coredump
1384         systemd-experimental
1385         systemd-homed
1386         systemd-journal-remote
1387         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1388         # systemd-network sub-package on openSUSE, has its own testsuite.
1389         systemd-network
1390         systemd-portable
1391         udev
1392     )
1393
1394     for p in "${pkgs[@]}"; do
1395         rpm -q "$p" &>/dev/null || continue
1396
1397         install_rpm "$p"
1398     done
1399
1400     dinfo "Install the data needed by the tests at runtime"
1401     inst_recursive "${SOURCE_DIR}/testdata"
1402     inst_recursive "${SOURCE_DIR}/unit-tests/manual"
1403
1404     # On openSUSE, this directory is not created at package install, at least
1405     # for now.
1406     mkdir -p "$initdir/var/log/journal/remote"
1407 }
1408
1409 install_fedora_systemd() {
1410     local required_packages=(
1411         systemd
1412         systemd-container
1413         systemd-libs
1414         systemd-pam
1415         systemd-tests
1416         systemd-udev
1417     )
1418     local optional_packages=(
1419         systemd-boot-unsigned
1420         systemd-bootchart
1421         systemd-journal-remote
1422         systemd-networkd
1423         systemd-oomd-defaults
1424         systemd-resolved
1425     )
1426     local package
1427
1428     for package in "${required_packages[@]}"; do
1429         install_rpm "$package"
1430     done
1431
1432     for package in "${optional_packages[@]}"; do
1433         rpm -q "$package" >/dev/null || continue
1434         install_rpm "$package"
1435     done
1436 }
1437
1438 install_distro_systemd() {
1439     dinfo "Install distro systemd"
1440
1441     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1442         install_debian_systemd
1443     elif get_bool "$LOOKS_LIKE_SUSE"; then
1444         install_suse_systemd
1445     elif get_bool "$LOOKS_LIKE_FEDORA"; then
1446         install_fedora_systemd
1447     else
1448         dfatal "NO_BUILD not supported for this distro"
1449         exit 1
1450     fi
1451 }
1452
1453 install_systemd() {
1454     dinfo "Install systemd"
1455     if get_bool "$NO_BUILD"; then
1456         install_distro_systemd
1457     else
1458         install_compiled_systemd
1459     fi
1460
1461     # Remove unneeded documentation
1462     rm -fr "${initdir:?}"/usr/share/{man,doc}
1463
1464     # Enable debug logging in PID1
1465     mkdir -p "$initdir/etc/systemd/system.conf.d/"
1466     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/system.conf.d/10-log-level.conf"
1467     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1468         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf.d/99-log-level.conf"
1469     fi
1470     # Enable debug logging for user instances as well
1471     mkdir -p "$initdir/etc/systemd/user.conf.d/"
1472     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/user.conf.d/10-log-level.conf"
1473     # Store coredumps in journal
1474     mkdir -p "$initdir/etc/systemd/coredump.conf.d/"
1475     echo -ne "[Coredump]\nStorage=journal\n" >"$initdir/etc/systemd/coredump.conf.d/10-storage-journal.conf"
1476     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1477     mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
1478     echo -ne "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/99-SYSTEMD_UNIT_PATH.conf"
1479
1480     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1481     # images, since it prevents gcov to write the coverage reports (*.gcda
1482     # files)
1483     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1484         mkdir -p "$initdir/etc/systemd/system/service.d/"
1485         echo -ne "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1486         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image to make the coverage work with
1487         # units using DynamicUser=yes. Do this only for services with test- prefix and a couple of
1488         # known-to-use DynamicUser=yes services, as setting this system-wide has many undesirable
1489         # side-effects, as it creates its own namespace.
1490         for service in test- systemd-journal-{gatewayd,upload}; do
1491             mkdir -p "$initdir/etc/systemd/system/$service.service.d/"
1492             echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/$service.service.d/99-gcov-rwpaths-override.conf"
1493         done
1494         # Ditto, but for the user daemon
1495         mkdir -p "$initdir/etc/systemd/user/test-.service.d/"
1496         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf"
1497         # Bind the $BUILD_DIR into nspawn containers that are executed using
1498         # machinectl. Unfortunately, the .nspawn files don't support drop-ins
1499         # so we have to inject the bind mount directly into
1500         # the systemd-nspawn@.service unit.
1501         cp "$initdir/usr/lib/systemd/system/systemd-nspawn@.service" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1502         sed -ri "s/^ExecStart=.+$/& --bind=${BUILD_DIR//\//\\\/}/" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1503         # Pass the $BUILD_DIR as $COVERAGE_BUILD_DIR env variable to the system
1504         # manager, similarly to what we do with $ASAN_RT_PATH during sanitized
1505         # builds
1506         mkdir -p "$initdir/etc/systemd/system.conf.d/"
1507         echo -ne "[Manager]\nDefaultEnvironment=COVERAGE_BUILD_DIR=$BUILD_DIR\n" >"$initdir/etc/systemd/system.conf.d/99-COVERAGE_BUILD_DIR.conf"
1508     fi
1509
1510     # If we're built with -Dportabled=false, tests with systemd-analyze
1511     # --profile will fail. Since we need just the profile (text) files, let's
1512     # copy them into the image if they don't exist there.
1513     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1514     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1515         dinfo "Couldn't find portable profiles in the test image"
1516         dinfo "Copying them directly from the source tree"
1517         mkdir -p "$portable_dir"
1518         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1519     fi
1520 }
1521
1522 get_ldpath() {
1523     local rpath
1524     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1525
1526     if [ -z "$rpath" ] ; then
1527         echo "$BUILD_DIR"
1528     else
1529         echo "$rpath"
1530     fi
1531 }
1532
1533 install_missing_libraries() {
1534     dinfo "Install missing libraries"
1535     # install possible missing libraries
1536     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/unit-tests/{,manual/,unsafe/}}*; do
1537         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1538     done
1539
1540     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1541     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1542     # if not present
1543     # See: https://github.com/systemd/systemd/pull/23858
1544     while read -r libgcc_s; do
1545         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1546     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1547
1548     local lib path
1549     # A number of dependencies is now optional via dlopen, so the install
1550     # script will not pick them up, since it looks at linkage.
1551     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1; do
1552         ddebug "Searching for $lib via pkg-config"
1553         if pkg-config --exists "$lib"; then
1554                 path="$(pkg-config --variable=libdir "$lib")"
1555                 if [ -z "${path}" ]; then
1556                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1557                     continue
1558                 fi
1559
1560                 if ! [[ ${lib} =~ ^lib ]]; then
1561                         lib="lib${lib}"
1562                 fi
1563                 # p11-kit-1's .so doesn't have the API level in the name
1564                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1565                         lib="libp11-kit"
1566                 fi
1567                 # Some pkg-config files are broken and give out the wrong paths
1568                 # (eg: libcryptsetup), so just ignore them
1569                 inst_libs "${path}/${lib}.so" || true
1570                 inst_library "${path}/${lib}.so" || true
1571
1572                 if [[ "$lib" == "libxkbcommon" ]]; then
1573                     install_x11_keymaps full
1574                 fi
1575         else
1576             ddebug "$lib.pc not found, skipping"
1577             continue
1578         fi
1579     done
1580
1581     # Install extra openssl 3 stuff
1582     path="$(pkg-config --variable=libdir libcrypto)"
1583     inst_simple "${path}/ossl-modules/legacy.so" || true
1584     inst_simple "${path}/ossl-modules/fips.so" || true
1585     inst_simple "${path}/engines-3/afalg.so" || true
1586     inst_simple "${path}/engines-3/capi.so" || true
1587     inst_simple "${path}/engines-3/loader_attic.so" || true
1588     inst_simple "${path}/engines-3/padlock.so" || true
1589
1590     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1591     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1592     # shellcheck disable=SC2046
1593     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1594 }
1595
1596 cleanup_loopdev() {
1597     if [ -n "${LOOPDEV:=}" ]; then
1598         ddebug "losetup -d $LOOPDEV"
1599         losetup -d "${LOOPDEV}"
1600         unset LOOPDEV
1601     fi
1602 }
1603
1604 add_at_exit_handler cleanup_loopdev
1605
1606 create_empty_image() {
1607     if [[ -z "${IMAGE_NAME:=}" ]]; then
1608         echo "create_empty_image: \$IMAGE_NAME not set"
1609         exit 1
1610     fi
1611
1612     # Partition sizes are in MiBs
1613     local root_size=768
1614     local data_size=100
1615     local esp_size=128
1616     local boot_size=128
1617     local total=
1618     if ! get_bool "$NO_BUILD"; then
1619         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1620             root_size=$((root_size + 200))
1621         fi
1622         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1623             root_size=$((root_size + 200))
1624         fi
1625         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1626             root_size=$((root_size + 250))
1627         fi
1628         if get_bool "$IS_BUILT_WITH_ASAN"; then
1629             root_size=$((root_size * 2))
1630         fi
1631     fi
1632
1633     if [[ "${IMAGE_ADDITIONAL_ROOT_SIZE:-0}" -gt 0 ]]; then
1634         root_size=$((root_size + IMAGE_ADDITIONAL_ROOT_SIZE))
1635     fi
1636     if [[ "${IMAGE_ADDITIONAL_DATA_SIZE:-0}" -gt 0 ]]; then
1637         data_size=$((data_size + IMAGE_ADDITIONAL_DATA_SIZE))
1638     fi
1639
1640     total=$((root_size + data_size + esp_size + boot_size))
1641
1642     echo "Setting up ${IMAGE_PUBLIC:?} (${total} MB)"
1643     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1644
1645     # Create the blank file to use as a root filesystem
1646     truncate -s "${total}M" "$IMAGE_PUBLIC"
1647
1648     LOOPDEV="$(losetup --show -P -f "$IMAGE_PUBLIC")"
1649     [[ -b "$LOOPDEV" ]] || return 1
1650     # Create two partitions - a root one and a data one (utilized by some tests)
1651     sfdisk "$LOOPDEV" <<EOF
1652 label: gpt
1653 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B name=esp  size=${esp_size}M
1654 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=root size=${root_size}M bootable
1655 type=BC13C2FF-59E6-4262-A352-B275FD6F7172 name=boot size=${boot_size}M
1656 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1657 EOF
1658
1659     udevadm settle
1660
1661     if ! mkfs -t vfat "${LOOPDEV}p1"; then
1662         dfatal "Failed to mkfs -t vfat ${LOOPDEV}p1"
1663         exit 1
1664     fi
1665
1666     local label=(-L systemd_boot)
1667     # mkfs.reiserfs doesn't know -L. so, use --label instead
1668     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1669     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p2" -q; then
1670         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p2 -q"
1671         exit 1
1672     fi
1673
1674     local label=(-L xbootldr)
1675     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label xbootldr)
1676     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p3" -q; then
1677         dfatal "Failed to mkfs -t ${FSTYPE} ${label[*]} ${LOOPDEV}p3 -q"
1678         exit 1
1679     fi
1680 }
1681
1682 mount_initdir() {
1683     if [ -z "${LOOPDEV:=}" ]; then
1684         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1685         LOOPDEV="$(losetup --show -P -f "$image")"
1686         [ -b "$LOOPDEV" ] || return 1
1687
1688         udevadm settle
1689     fi
1690
1691     if ! mountpoint -q "${initdir:?}"; then
1692         mkdir -p "$initdir"
1693         mount "${LOOPDEV}p2" "$initdir"
1694         TEST_SETUP_CLEANUP_ROOTDIR=1
1695     fi
1696 }
1697
1698 cleanup_initdir() {
1699     # only umount if create_empty_image_rootdir() was called to mount it
1700     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1701         _umount_dir "${initdir:?}"
1702     fi
1703 }
1704
1705 umount_loopback() {
1706     # unmount the loopback device from all places. Otherwise we risk file
1707     # system corruption.
1708     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1709         ddebug "Unmounting all uses of $device"
1710         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1711     done
1712 }
1713
1714 create_empty_image_rootdir() {
1715     create_empty_image
1716     mount_initdir
1717 }
1718
1719 check_asan_reports() {
1720     local ret=0
1721     local root="${1:?}"
1722     local log report
1723
1724     if get_bool "$IS_BUILT_WITH_ASAN"; then
1725         ls -l "$root"
1726         if [[ -e "$root/systemd.asan.log.1" ]]; then
1727             cat "$root/systemd.asan.log.1"
1728             ret=$((ret+1))
1729         fi
1730
1731         for log in pid1 journald; do
1732             report="$(find "$root" -name "systemd-$log.*san.log*" -exec cat {} \;)"
1733             if [[ -n "$report" ]]; then
1734                 printf "%s\n" "$report"
1735                 # shellcheck disable=SC2015
1736                 [[ "$log" == journald ]] && cat "$root/systemd-journald.out" || :
1737                 ret=$((ret+1))
1738             fi
1739         done
1740
1741         # May 08 13:23:31 H testleak[2907148]: SUMMARY: AddressSanitizer: 4 byte(s) leaked in 1 allocation(s).
1742         pids="$(
1743             "$JOURNALCTL" -D "$root/var/log/journal" --grep 'SUMMARY: .*Sanitizer:' |
1744                    grep -v -E 'dbus-daemon|dbus-broker-launch' |
1745                    sed -r -n 's/.* .+\[([0-9]+)\]: SUMMARY:.*/\1/p'
1746         )"
1747
1748         if [[ -n "$pids" ]]; then
1749             ret=$((ret+1))
1750             for pid in $pids; do
1751                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1752             done
1753         fi
1754     fi
1755
1756     return $ret
1757 }
1758
1759 check_coverage_reports() {
1760     local root="${1:?}"
1761
1762     if get_bool "$NO_BUILD"; then
1763         return 0
1764     fi
1765     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1766         return 0
1767     fi
1768
1769     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1770         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1771     else
1772         dest="${TESTDIR:?}/coverage-info"
1773     fi
1774
1775     if [[ ! -e "${TESTDIR:?}/coverage-base" ]]; then
1776         # This shouldn't happen, as the report is generated during the setup
1777         # phase (test_setup()).
1778         derror "Missing base coverage report"
1779         return 1
1780     fi
1781
1782     # Create a coverage report that will later be uploaded. Remove info about system
1783     # libraries/headers and generated files, as we don't really care about them.
1784     lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1785     if [[ -f "$dest" ]]; then
1786         # If the destination report file already exists, don't overwrite it, but
1787         # merge it with the already present one - this usually happens when
1788         # running both "parts" of a test in one run (the qemu and the nspawn part).
1789         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1790     else
1791         # If there's no prior coverage report, merge the new one with the base
1792         # report we did during the setup phase (see test_setup()).
1793         lcov --add-tracefile "${TESTDIR:?}/coverage-base" --add-tracefile "${dest}.new" -o "${dest}"
1794     fi
1795     lcov --remove "$dest" -o "$dest" '/usr/include/*' '/usr/lib/*' "${BUILD_DIR:?}/*"
1796     rm -f "${dest}.new"
1797
1798     # If the test logs contain lines like:
1799     #
1800     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1801     #
1802     # it means we're possibly missing some coverage since gcov can't write the stats,
1803     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1804     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1805     # `setfacl` stuff in install_compiled_systemd().
1806     #
1807     # Also, a note: some tests, like TEST-46, overmount /home with tmpfs, which
1808     # means if your build dir is under /home/your-user (which is usually the
1809     # case) you might get bogus errors and missing coverage.
1810     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1811        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1812         derror "Detected possibly missing coverage, check the journal"
1813         return 1
1814     fi
1815
1816     return 0
1817 }
1818
1819 save_journal() {
1820     local source_dir="${1:?}"
1821     local state="${2:?}"
1822     # Default to always saving journal
1823     local save="yes"
1824     local dest_dir dest_name dest
1825
1826     if [[ "${TEST_SAVE_JOURNAL:-}" == "no" ]]; then
1827         save="no"
1828     elif [[ "${TEST_SAVE_JOURNAL:-}" == "fail" && "$state" -eq 0 ]]; then
1829         save="no"
1830     fi
1831
1832     if [[ -n "${ARTIFACT_DIRECTORY:-}" ]]; then
1833         dest_dir="$ARTIFACT_DIRECTORY"
1834         dest_name="${testname:?}.journal"
1835     else
1836         dest_dir="${TESTDIR:?}"
1837         dest_name="system.journal"
1838     fi
1839
1840     # Show messages from the testsuite-XX.service or messages with priority "warning" and higher
1841     echo " --- $source_dir ---"
1842     "$JOURNALCTL" --no-pager --no-hostname -o short-monotonic -D "$source_dir" \
1843         _SYSTEMD_UNIT="testsuite-${TESTID:?}.service" + PRIORITY=4 + PRIORITY=3 + PRIORITY=2 + PRIORITY=1 + PRIORITY=0
1844
1845     if get_bool "$save"; then
1846         # If we don't have systemd-journal-remote copy all journals from /var/log/journal/
1847         # to $dest_dir/journals/ as is, otherwise merge all journals into a single .journal
1848         # file
1849         if [[ -z "${SYSTEMD_JOURNAL_REMOTE:-}" ]]; then
1850             dest="$dest_dir/journals"
1851             mkdir -p "$dest"
1852             cp -a "$source_dir/*" "$dest/"
1853         else
1854             dest="$dest_dir/$dest_name"
1855             "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $source_dir"
1856         fi
1857
1858         if [[ -n "${SUDO_USER:-}" ]]; then
1859             setfacl -R -m "user:$SUDO_USER:r-X" "$dest"
1860         fi
1861
1862         # we want to print this sometime later, so save this in a variable
1863         JOURNAL_LIST="$(ls -lR "$dest")"
1864     fi
1865
1866     rm -rf "${source_dir:?}"/*
1867 }
1868
1869 check_result_common() {
1870     local workspace="${1:?}"
1871     local ret
1872
1873     if [ -s "$workspace/failed" ]; then
1874         # Non-empty …/failed has highest priority
1875         cp -a "$workspace/failed" "${TESTDIR:?}/"
1876         if [ -n "${SUDO_USER}" ]; then
1877             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1878         fi
1879         ret=1
1880     elif get_bool "$TIMED_OUT"; then
1881         echo "(timeout)" >"${TESTDIR:?}/failed"
1882         ret=2
1883     elif [ -e "$workspace/testok" ]; then
1884         # …/testok always counts (but with lower priority than …/failed)
1885         ret=0
1886     elif [ -e "$workspace/skipped" ]; then
1887         # …/skipped always counts (a message is expected)
1888         echo "${TESTNAME:?} was skipped:"
1889         cat "$workspace/skipped"
1890         ret=0
1891     else
1892         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1893         ret=3
1894     fi
1895
1896     check_asan_reports "$workspace" || ret=4
1897
1898     check_coverage_reports "$workspace" || ret=5
1899
1900     save_journal "$workspace/var/log/journal" $ret
1901
1902     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1903         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1904     fi
1905
1906     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1907         echo -n "${TESTNAME:?}: "
1908         cat "$TESTDIR/failed"
1909     fi
1910     echo "${JOURNAL_LIST:-"No journals were saved"}"
1911
1912     return ${ret:?}
1913 }
1914
1915 check_result_nspawn() {
1916     local workspace="${1:?}"
1917     local ret=0
1918
1919     # Run a test-specific checks if defined by check_result_nspawn_hook()
1920     if declare -F check_result_nspawn_hook >/dev/null; then
1921         if ! check_result_nspawn_hook "${workspace}"; then
1922             derror "check_result_nspawn_hook() returned with EC > 0"
1923             ret=4
1924         fi
1925     fi
1926
1927     check_result_common "${workspace}" || ret=$?
1928
1929     _umount_dir "${initdir:?}"
1930
1931     return $ret
1932 }
1933
1934 # can be overridden in specific test
1935 check_result_qemu() {
1936     local ret=0
1937     mount_initdir
1938
1939     # Run a test-specific checks if defined by check_result_qemu_hook()
1940     if declare -F check_result_qemu_hook >/dev/null; then
1941         if ! check_result_qemu_hook "${initdir:?}"; then
1942             derror "check_result_qemu_hook() returned with EC > 0"
1943             ret=4
1944         fi
1945     fi
1946
1947     check_result_common "${initdir:?}" || ret=$?
1948
1949     _umount_dir "${initdir:?}"
1950
1951     return $ret
1952 }
1953
1954 check_result_nspawn_unittests() {
1955     local workspace="${1:?}"
1956     local ret=1
1957
1958     [[ -e "$workspace/testok" ]] && ret=0
1959
1960     if [[ -s "$workspace/failed" ]]; then
1961         ret=$((ret + 1))
1962         echo "=== Failed test log ==="
1963         cat "$workspace/failed"
1964     else
1965         if [[ -s "$workspace/skipped" ]]; then
1966             echo "=== Skipped test log =="
1967             cat "$workspace/skipped"
1968             # We might have only skipped tests - that should not fail the job
1969             ret=0
1970         fi
1971         if [[ -s "$workspace/testok" ]]; then
1972             echo "=== Passed tests ==="
1973             cat "$workspace/testok"
1974         fi
1975     fi
1976
1977     get_bool "${TIMED_OUT:=}" && ret=1
1978     check_coverage_reports "$workspace" || ret=5
1979
1980     save_journal "$workspace/var/log/journal" $ret
1981     echo "${JOURNAL_LIST:-"No journals were saved"}"
1982
1983     _umount_dir "${initdir:?}"
1984
1985     return $ret
1986 }
1987
1988 check_result_qemu_unittests() {
1989     local ret=1
1990
1991     mount_initdir
1992     [[ -e "${initdir:?}/testok" ]] && ret=0
1993
1994     if [[ -s "$initdir/failed" ]]; then
1995         ret=$((ret + 1))
1996         echo "=== Failed test log ==="
1997         cat "$initdir/failed"
1998     else
1999         if [[ -s "$initdir/skipped" ]]; then
2000             echo "=== Skipped test log =="
2001             cat "$initdir/skipped"
2002             # We might have only skipped tests - that should not fail the job
2003             ret=0
2004         fi
2005         if [[ -s "$initdir/testok" ]]; then
2006             echo "=== Passed tests ==="
2007             cat "$initdir/testok"
2008         fi
2009     fi
2010
2011     get_bool "${TIMED_OUT:=}" && ret=1
2012     check_coverage_reports "$initdir" || ret=5
2013
2014     save_journal "$initdir/var/log/journal" $ret
2015     echo "${JOURNAL_LIST:-"No journals were saved"}"
2016
2017     _umount_dir "$initdir"
2018
2019     return $ret
2020 }
2021
2022 create_rc_local() {
2023     dinfo "Create rc.local"
2024     mkdir -p "${initdir:?}/etc/rc.d"
2025     cat >"$initdir/etc/rc.d/rc.local" <<EOF
2026 #!/usr/bin/env bash
2027 exit 0
2028 EOF
2029     chmod 0755 "$initdir/etc/rc.d/rc.local"
2030 }
2031
2032 install_execs() {
2033     ddebug "Install executables from the service files"
2034
2035     local pkg_config_path="${BUILD_DIR:?}/src/core/"
2036     local systemunitdir userunitdir exe
2037     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
2038     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
2039     while read -r exe; do
2040         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
2041         # also, plymouth is pulled in by rescue.service, but even there the exit code
2042         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
2043         dinfo "Attempting to install $exe (based on unit file reference)"
2044         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
2045     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
2046 }
2047
2048 generate_module_dependencies() {
2049     dinfo "Generate modules dependencies"
2050     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
2051         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
2052             dfatal "\"depmod -a $KERNEL_VER\" failed."
2053             exit 1
2054     fi
2055 }
2056
2057 install_depmod_files() {
2058     dinfo "Install depmod files"
2059     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
2060     inst "/lib/modules/$KERNEL_VER/modules.builtin"
2061 }
2062
2063 install_plymouth() {
2064     dinfo "Install plymouth"
2065     # install plymouth, if found... else remove plymouth service files
2066     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
2067     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
2068     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
2069     #         image_install plymouth plymouthd
2070     # else
2071         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
2072     # fi
2073 }
2074
2075 install_haveged() {
2076     # If haveged is installed, it's probably included in initrd and needs to be
2077     # installed in the image too.
2078     if [ -x /usr/sbin/haveged ]; then
2079         dinfo "Install haveged files"
2080         inst /usr/sbin/haveged
2081         for u in /usr/lib/systemd/system/haveged*; do
2082             inst "$u"
2083         done
2084     fi
2085 }
2086
2087 install_ld_so_conf() {
2088     dinfo "Install /etc/ld.so.conf*"
2089     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
2090     ldconfig -r "$initdir"
2091 }
2092
2093 install_testuser() {
2094     dinfo "Set up a test user"
2095     # create unprivileged user for user manager tests
2096     mkdir -p "${initdir:?}/etc/sysusers.d"
2097     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
2098 u testuser    4711     "Test User" /home/testuser
2099 EOF
2100
2101     mkdir -p "$initdir/home/testuser"
2102     chmod 0700 "$initdir/home/testuser"
2103     chown 4711:4711 "$initdir/home/testuser"
2104 }
2105
2106 install_config_files() {
2107     dinfo "Install config files"
2108     inst /etc/sysconfig/init || :
2109     inst /etc/passwd
2110     inst /etc/shadow
2111     inst_any /etc/login.defs /usr/etc/login.defs
2112     inst /etc/group
2113     inst /etc/shells
2114     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
2115     inst /etc/pam.conf || :
2116     inst_any /etc/os-release /usr/lib/os-release
2117     inst /etc/localtime
2118     # we want an empty environment
2119     : >"${initdir:?}/etc/environment"
2120     : >"$initdir/etc/machine-id"
2121     : >"$initdir/etc/resolv.conf"
2122
2123     # set the hostname
2124     echo 'H' >"$initdir/etc/hostname"
2125
2126     # let's set up just one image with the traditional verbose output
2127     if [ "${IMAGE_NAME:?}" != "basic" ]; then
2128         mkdir -p "$initdir/etc/systemd/system.conf.d"
2129         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
2130     fi
2131 }
2132
2133 install_basic_tools() {
2134     dinfo "Install basic tools"
2135     image_install "${BASICTOOLS[@]}"
2136     image_install -o sushell
2137     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
2138     image_install -o ldconfig.real
2139 }
2140
2141 install_debug_tools() {
2142     dinfo "Install debug tools"
2143     image_install -o "${DEBUGTOOLS[@]}"
2144
2145     if get_bool "$INTERACTIVE_DEBUG"; then
2146         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
2147         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
2148         mkdir -p "$getty_override"
2149         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
2150         echo 'export TERM=linux' >>"$initdir/etc/profile"
2151
2152         if command -v resize >/dev/null; then
2153             image_install resize
2154             echo "resize" >>"$initdir/etc/profile"
2155         fi
2156
2157         # Sometimes we might end up with plymouthd still running (especially
2158         # with the initrd -> asan_wrapper -> systemd transition), which will eat
2159         # our inputs and make debugging via tty impossible. Let's fix this by
2160         # killing plymouthd explicitly for the interactive sessions.
2161         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
2162         #   - https://github.com/llvm/llvm-project/issues/49223
2163         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
2164         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
2165         cat >"$plymouth_unit" <<EOF
2166 [Unit]
2167 After=multi-user.target
2168
2169 [Service]
2170 ExecStart=sh -c 'killall --verbose plymouthd || :'
2171
2172 [Install]
2173 WantedBy=multi-user.target
2174 EOF
2175         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
2176     fi
2177 }
2178
2179 install_libnss() {
2180     dinfo "Install libnss"
2181     # install libnss_files for login
2182     local NSS_LIBS
2183     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
2184     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
2185         image_install "${NSS_LIBS[@]}"
2186     fi
2187 }
2188
2189 install_dbus() {
2190     dinfo "Install dbus"
2191     inst "${ROOTLIBDIR:?}/system/dbus.socket"
2192
2193     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2194     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
2195         inst "$ROOTLIBDIR/system/dbus-broker.service"
2196         inst_symlink /etc/systemd/system/dbus.service
2197         inst /usr/bin/dbus-broker
2198         inst /usr/bin/dbus-broker-launch
2199     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
2200         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2201         inst "$ROOTLIBDIR/system/dbus-daemon.service"
2202         # Alias symlink
2203         inst_symlink /etc/systemd/system/dbus.service
2204     else
2205         inst "$ROOTLIBDIR/system/dbus.service"
2206     fi
2207
2208     while read -r file; do
2209         inst "$file"
2210     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
2211
2212     # setup policy for Type=dbus test
2213     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
2214     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
2215 <?xml version="1.0"?>
2216 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2217         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2218 <busconfig>
2219     <policy user="root">
2220         <allow own="systemd.test.ExecStopPost"/>
2221     </policy>
2222 </busconfig>
2223 EOF
2224
2225     # If we run without KVM, bump the service start timeout
2226     if ! get_bool "$QEMU_KVM"; then
2227         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
2228 <?xml version="1.0"?>
2229 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
2230         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
2231 <busconfig>
2232     <limit name="service_start_timeout">120000</limit>
2233 </busconfig>
2234 EOF
2235         # Bump the client-side timeout in sd-bus as well
2236         mkdir -p "$initdir/etc/systemd/system.conf.d"
2237         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=120' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
2238     fi
2239 }
2240
2241 install_user_dbus() {
2242     dinfo "Install user dbus"
2243     local userunitdir
2244     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
2245         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
2246         userunitdir=/usr/lib/systemd/user
2247     fi
2248
2249     inst "$userunitdir/dbus.socket"
2250     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
2251
2252     # Append the After= dependency on dbus in case it isn't already set up
2253     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
2254     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
2255 [Unit]
2256 After=dbus.service
2257 EOF
2258
2259     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
2260     if [ -f "$userunitdir/dbus-broker.service" ]; then
2261         inst "$userunitdir/dbus-broker.service"
2262         inst_symlink /etc/systemd/user/dbus.service
2263     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
2264         # Fedora rawhide replaced dbus.service with dbus-daemon.service
2265         inst "$userunitdir/dbus-daemon.service"
2266         # Alias symlink
2267         inst_symlink /etc/systemd/user/dbus.service
2268     else
2269         inst "$userunitdir/dbus.service"
2270     fi
2271 }
2272
2273 install_pam() {
2274     dinfo "Install PAM"
2275     local paths=()
2276
2277     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
2278         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
2279     else
2280         paths+=(/lib*/security)
2281     fi
2282
2283     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
2284         [ -d "$d" ] && paths+=("$d")
2285     done
2286
2287     while read -r file; do
2288         inst "$file"
2289     done < <(find "${paths[@]}" -xtype f)
2290
2291     # pam_unix depends on unix_chkpwd.
2292     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2293     image_install -o unix_chkpwd
2294
2295     # set empty root password for easy debugging
2296     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2297
2298     # And make sure pam_unix will accept it by making sure that
2299     # the PAM module has the nullok option.
2300     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2301         [ -d "$initdir/$d" ] || continue
2302         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2303     done
2304 }
2305
2306 install_locales() {
2307     # install only C.UTF-8 and English locales
2308     dinfo "Install locales"
2309
2310     if command -v meson >/dev/null \
2311             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2312             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2313         # locale-gen support
2314         image_install -o locale-gen localedef
2315         inst /etc/locale.gen || :
2316         inst /usr/share/i18n/SUPPORTED || :
2317         inst_recursive /usr/share/i18n/charmaps
2318         inst_recursive /usr/share/i18n/locales
2319         inst_recursive /usr/share/locale/en*
2320         inst_recursive /usr/share/locale/de*
2321         image_install /usr/share/locale/locale.alias
2322         # locale-gen might either generate each locale separately or merge them
2323         # into a single archive
2324         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2325               image_install /usr/lib/locale/locale-archive); then
2326             dfatal "Failed to install required locales"
2327             exit 1
2328         fi
2329     else
2330         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2331     fi
2332 }
2333
2334 # shellcheck disable=SC2120
2335 install_keymaps() {
2336     local i p
2337     local -a prefix=(
2338         "/usr/lib"
2339         "/usr/share"
2340     )
2341
2342     dinfo "Install console keymaps"
2343
2344     if (( $# == 0 )); then
2345         for p in "${prefix[@]}"; do
2346             # The first three paths may be deprecated.
2347             # It seems now the last three paths are used by many distributions.
2348             for i in \
2349                 "$p"/kbd/keymaps/include/* \
2350                 "$p"/kbd/keymaps/i386/include/* \
2351                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2352                 "$p"/kbd/keymaps/legacy/include/* \
2353                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2354                 "$p"/kbd/keymaps/xkb/us*; do
2355                     [[ -f "$i" ]] || continue
2356                     inst "$i"
2357             done
2358         done
2359     else
2360         # When it takes any argument, then install more keymaps.
2361         for p in "${prefix[@]}"; do
2362             for i in \
2363                 "$p"/kbd/keymaps/include/* \
2364                 "$p"/kbd/keymaps/i386/*/* \
2365                 "$p"/kbd/keymaps/legacy/i386/*/* \
2366                 "$p"/kbd/keymaps/xkb/*; do
2367                     [[ -f "$i" ]] || continue
2368                     inst "$i"
2369             done
2370         done
2371     fi
2372 }
2373
2374 install_x11_keymaps() {
2375     dinfo "Install x11 keymaps"
2376
2377     if (( $# == 0 )); then
2378         # Install only keymap list.
2379         inst /usr/share/X11/xkb/rules/base.lst
2380     else
2381         # When it takes any argument, then install all keymaps.
2382         inst_recursive /usr/share/X11/xkb
2383     fi
2384 }
2385
2386 install_zoneinfo() {
2387     dinfo "Install time zones"
2388     inst_any /usr/share/zoneinfo/Asia/Seoul
2389     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2390     inst_any /usr/share/zoneinfo/Australia/Sydney
2391     inst_any /usr/share/zoneinfo/Europe/Berlin
2392     inst_any /usr/share/zoneinfo/Europe/Dublin
2393     inst_any /usr/share/zoneinfo/Europe/Kiev
2394     inst_any /usr/share/zoneinfo/Pacific/Auckland
2395     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2396     inst_any /usr/share/zoneinfo/CET
2397     inst_any /usr/share/zoneinfo/EET
2398     inst_any /usr/share/zoneinfo/UTC
2399 }
2400
2401 install_fonts() {
2402     dinfo "Install system fonts"
2403     for i in \
2404         /usr/{lib,share}/kbd/consolefonts/eurlatgr* \
2405         /usr/{lib,share}/kbd/consolefonts/latarcyrheb-sun16*; do
2406             [[ -f "$i" ]] || continue
2407             inst "$i"
2408     done
2409 }
2410
2411 install_terminfo() {
2412     dinfo "Install terminfo files"
2413     local terminfodir
2414     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2415         [ -f "${terminfodir}/l/linux" ] && break
2416     done
2417     image_install -o "${terminfodir}/l/linux"
2418 }
2419
2420 has_user_dbus_socket() {
2421     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2422         return 0
2423     else
2424         echo "Per-user instances are not supported. Skipping..."
2425         return 1
2426     fi
2427 }
2428
2429 setup_nspawn_root_hook() { :;}
2430
2431 setup_nspawn_root() {
2432     if [ -z "${initdir}" ]; then
2433         dfatal "\$initdir not defined"
2434         exit 1
2435     fi
2436
2437     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2438
2439     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2440         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2441         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2442     fi
2443
2444     setup_nspawn_root_hook
2445 }
2446
2447 setup_basic_dirs() {
2448     mkdir -p "${initdir:?}/run"
2449     mkdir -p "$initdir/etc/systemd/system"
2450     mkdir -p "$initdir/var/log/journal"
2451
2452
2453     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2454         if [ -L "/$d" ]; then
2455             inst_symlink "/$d"
2456         else
2457             inst_dir "/$d"
2458         fi
2459     done
2460
2461     ln -sfn /run "$initdir/var/run"
2462     ln -sfn /run/lock "$initdir/var/lock"
2463 }
2464
2465 mask_supporting_services() {
2466     # mask some services that we do not want to run in these tests
2467     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2468     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2469     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2470     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2471     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2472 }
2473
2474 inst_libs() {
2475     local bin="${1:?}"
2476     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2477     local file line
2478
2479     while read -r line; do
2480         [[ "$line" = 'not a dynamic executable' ]] && break
2481         # Ignore errors about our own stuff missing. This is most likely caused
2482         # by ldd attempting to use the unprefixed RPATH.
2483         [[ "$line" =~ (libsystemd|libudev).*\ not\ found ]] && continue
2484
2485         if [[ "$line" =~ not\ found ]]; then
2486             dfatal "Missing a shared library required by $bin."
2487             dfatal "Run \"ldd $bin\" to find out what it is."
2488             dfatal "$line"
2489             dfatal "Cannot create a test image."
2490             exit 1
2491         fi
2492
2493         if [[ "$line" =~ $so_regex ]]; then
2494             file="${BASH_REMATCH[1]}"
2495             [[ -e "${initdir:?}/$file" ]] && continue
2496             inst_library "$file"
2497         fi
2498     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2499 }
2500
2501 import_testdir() {
2502     # make sure we don't get a stale LOOPDEV value from old times
2503     local _LOOPDEV="${LOOPDEV:=}"
2504     # We don't want shellcheck to follow & check the $STATEFILE
2505     # shellcheck source=/dev/null
2506     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2507     LOOPDEV="$_LOOPDEV"
2508     if [[ ! -d "$TESTDIR" ]]; then
2509         if [[ -z "$TESTDIR" ]]; then
2510             TESTDIR="$(mktemp --tmpdir=$WORKDIR -d -t systemd-test.XXXXXX)"
2511         else
2512             mkdir -p "$TESTDIR"
2513         fi
2514
2515         cat >"$STATEFILE" <<EOF
2516 TESTDIR="$TESTDIR"
2517 EOF
2518         export TESTDIR
2519     fi
2520
2521     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2522     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2523 }
2524
2525 import_initdir() {
2526     initdir="${TESTDIR:?}/root"
2527     mkdir -p "$initdir"
2528     export initdir
2529 }
2530
2531 get_cgroup_hierarchy() {
2532     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2533         cgroup2fs)
2534             echo "unified"
2535             ;;
2536         tmpfs)
2537             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2538                 echo "hybrid"
2539             else
2540                 echo "legacy"
2541             fi
2542             ;;
2543         *)
2544             dfatal "Failed to determine host's cgroup hierarchy"
2545             exit 1
2546     esac
2547 }
2548
2549 ## @brief Converts numeric logging level to the first letter of level name.
2550 #
2551 # @param lvl Numeric logging level in range from 1 to 6.
2552 # @retval 1 if @a lvl is out of range.
2553 # @retval 0 if @a lvl is correct.
2554 # @result Echoes first letter of level name.
2555 _lvl2char() {
2556     case "$1" in
2557         1) echo F;;
2558         2) echo E;;
2559         3) echo W;;
2560         4) echo I;;
2561         5) echo D;;
2562         6) echo T;;
2563         *) return 1;;
2564     esac
2565 }
2566
2567 ## @brief Internal helper function for _do_dlog()
2568 #
2569 # @param lvl Numeric logging level.
2570 # @param msg Message.
2571 # @retval 0 It's always returned, even if logging failed.
2572 #
2573 # @note This function is not supposed to be called manually. Please use
2574 # dtrace(), ddebug(), or others instead which wrap this one.
2575 #
2576 # This function calls _do_dlog() either with parameter msg, or if
2577 # none is given, it will read standard input and will use every line as
2578 # a message.
2579 #
2580 # This enables:
2581 # dwarn "This is a warning"
2582 # echo "This is a warning" | dwarn
2583 LOG_LEVEL="${LOG_LEVEL:-4}"
2584
2585 dlog() {
2586     local lvl lvlc
2587
2588     [ -z "$LOG_LEVEL" ] && return 0
2589     lvl="${1:?}"; shift
2590     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2591     lvlc="$(_lvl2char "$lvl")" || return 0
2592
2593     if [ $# -ge 1 ]; then
2594         echo "$lvlc: $*"
2595     else
2596         while read -r line; do
2597             echo "$lvlc: " "$line"
2598         done
2599     fi
2600 }
2601
2602 ## @brief Logs message at TRACE level (6)
2603 #
2604 # @param msg Message.
2605 # @retval 0 It's always returned, even if logging failed.
2606 dtrace() {
2607     set +x
2608     dlog 6 "$@"
2609     if get_bool "${debug:=}"; then
2610         set -x
2611     fi
2612 }
2613
2614 ## @brief Logs message at DEBUG level (5)
2615 #
2616 # @param msg Message.
2617 # @retval 0 It's always returned, even if logging failed.
2618 ddebug() {
2619     dlog 5 "$@"
2620 }
2621
2622 ## @brief Logs message at INFO level (4)
2623 #
2624 # @param msg Message.
2625 # @retval 0 It's always returned, even if logging failed.
2626 dinfo() {
2627     set +x
2628     dlog 4 "$@"
2629     if get_bool "${debug:=}"; then
2630         set -x
2631     fi
2632 }
2633
2634 ## @brief Logs message at WARN level (3)
2635 #
2636 # @param msg Message.
2637 # @retval 0 It's always returned, even if logging failed.
2638 dwarn() {
2639     set +x
2640     dlog 3 "$@"
2641     if get_bool "${debug:=}"; then
2642         set -x
2643     fi
2644 }
2645
2646 ## @brief Logs message at ERROR level (2)
2647 #
2648 # @param msg Message.
2649 # @retval 0 It's always returned, even if logging failed.
2650 derror() {
2651     dlog 2 "$@"
2652 }
2653
2654 ## @brief Logs message at FATAL level (1)
2655 #
2656 # @param msg Message.
2657 # @retval 0 It's always returned, even if logging failed.
2658 dfatal() {
2659     set +x
2660     dlog 1 "$@"
2661     if get_bool "${debug:=}"; then
2662         set -x
2663     fi
2664 }
2665
2666
2667 # Generic substring function.  If $2 is in $1, return 0.
2668 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2669
2670 # normalize_path <path>
2671 # Prints the normalized path, where it removes any duplicated
2672 # and trailing slashes.
2673 # Example:
2674 # $ normalize_path ///test/test//
2675 # /test/test
2676 normalize_path() {
2677     shopt -q -s extglob
2678     set -- "${1//+(\/)//}"
2679     shopt -q -u extglob
2680     echo "${1%/}"
2681 }
2682
2683 # convert_abs_rel <from> <to>
2684 # Prints the relative path, when creating a symlink to <to> from <from>.
2685 # Example:
2686 # $ convert_abs_rel /usr/bin/test /bin/test-2
2687 # ../../bin/test-2
2688 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2689 convert_abs_rel() {
2690     local __current __absolute __abssize __cursize __newpath
2691     local -i __i __level
2692
2693     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2694
2695     # corner case #1 - self looping link
2696     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2697
2698     # corner case #2 - own dir link
2699     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2700
2701     IFS="/" read -ra __current <<< "$1"
2702     IFS="/" read -ra __absolute <<< "$2"
2703
2704     __abssize=${#__absolute[@]}
2705     __cursize=${#__current[@]}
2706
2707     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2708         (( __level++ ))
2709         if (( __level > __abssize || __level > __cursize ))
2710         then
2711             break
2712         fi
2713     done
2714
2715     for ((__i = __level; __i < __cursize-1; __i++)); do
2716         if ((__i > __level))
2717         then
2718             __newpath=$__newpath"/"
2719         fi
2720         __newpath=$__newpath".."
2721     done
2722
2723     for ((__i = __level; __i < __abssize; __i++)); do
2724         if [[ -n $__newpath ]]
2725         then
2726             __newpath=$__newpath"/"
2727         fi
2728         __newpath=$__newpath${__absolute[__i]}
2729     done
2730
2731     echo "$__newpath"
2732 }
2733
2734
2735 # Install a directory, keeping symlinks as on the original system.
2736 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2737 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2738 # and a symlink ${initdir}/lib -> lib64.
2739 inst_dir() {
2740     local dir="${1:?}"
2741     local part="${dir%/*}"
2742     local file
2743
2744     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2745
2746     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2747         dir="$part $dir"
2748         part="${part%/*}"
2749     done
2750
2751     # iterate over parent directories
2752     for file in $dir; do
2753         [[ -e "${initdir}/$file" ]] && continue
2754         if [[ -L $file ]]; then
2755             inst_symlink "$file"
2756         else
2757             # create directory
2758             mkdir -m 0755 "${initdir}/$file" || return 1
2759             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2760             chmod u+w "${initdir}/$file"
2761         fi
2762     done
2763 }
2764
2765 # $1 = file to copy to ramdisk
2766 # $2 (optional) Name for the file on the ramdisk
2767 # Location of the image dir is assumed to be $initdir
2768 # We never overwrite the target if it exists.
2769 inst_simple() {
2770     [[ -f "${1:?}" ]] || return 1
2771     strstr "$1" "/" || return 1
2772
2773     local src="$1"
2774     local target="${2:-$1}"
2775     if ! [[ -d ${initdir:?}/$target ]]; then
2776         [[ -e ${initdir}/$target ]] && return 0
2777         [[ -L ${initdir}/$target ]] && return 0
2778         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2779     fi
2780     # install checksum files also
2781     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2782         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2783     fi
2784     ddebug "Installing $src"
2785     cp --sparse=always --force --dereference --preserve=all "$src" "${initdir}/$target"
2786 }
2787
2788 # find symlinks linked to given library file
2789 # $1 = library file
2790 # Function searches for symlinks by stripping version numbers appended to
2791 # library filename, checks if it points to the same target and finally
2792 # prints the list of symlinks to stdout.
2793 #
2794 # Example:
2795 # rev_lib_symlinks libfoo.so.8.1
2796 # output: libfoo.so.8 libfoo.so
2797 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2798 rev_lib_symlinks() {
2799     local fn="${1:?}"
2800     local links=""
2801     local orig
2802     orig="$(readlink -f "$1")"
2803
2804     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2805
2806     until [[ "${fn##*.}" == so ]]; do
2807         fn="${fn%.*}"
2808         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2809     done
2810
2811     echo "${links}"
2812 }
2813
2814 # Same as above, but specialized to handle dynamic libraries.
2815 # It handles making symlinks according to how the original library
2816 # is referenced.
2817 inst_library() {
2818     local src="${1:?}"
2819     local dest="${2:-$1}"
2820     local reallib symlink
2821
2822     strstr "$1" "/" || return 1
2823     [[ -e ${initdir:?}/$dest ]] && return 0
2824     if [[ -L $src ]]; then
2825         # install checksum files also
2826         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2827             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2828         fi
2829         reallib="$(readlink -f "$src")"
2830         inst_simple "$reallib" "$reallib"
2831         inst_dir "${dest%/*}"
2832         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2833         ddebug "Creating symlink $reallib -> $dest"
2834         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2835     else
2836         inst_simple "$src" "$dest"
2837     fi
2838
2839     # Create additional symlinks.  See rev_symlinks description.
2840     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2841         if [[ ! -e "$initdir/$symlink" ]]; then
2842             ddebug "Creating extra symlink: $symlink"
2843             inst_symlink "$symlink"
2844         fi
2845     done
2846 }
2847
2848 # find a binary.  If we were not passed the full path directly,
2849 # search in the usual places to find the binary.
2850 find_binary() {
2851     local bin="${1:?}"
2852     if [[ -z ${bin##/*} ]]; then
2853         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2854             echo "$bin"
2855             return 0
2856         fi
2857     fi
2858
2859     type -P "$bin"
2860 }
2861
2862 # Same as above, but specialized to install binary executables.
2863 # Install binary executable, and all shared library dependencies, if any.
2864 inst_binary() {
2865     local bin="${1:?}"
2866     local path target
2867
2868     # In certain cases we might attempt to install a binary which is already
2869     # present in the test image, yet it's missing from the host system.
2870     # In such cases, let's check if the binary indeed exists in the image
2871     # before doing any other checks. If it does, immediately return with
2872     # success.
2873     if [[ $# -eq 1 ]]; then
2874         for path in "" bin sbin usr/bin usr/sbin; do
2875             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2876         done
2877     fi
2878
2879     bin="$(find_binary "$bin")" || return 1
2880     target="${2:-$bin}"
2881     [[ -e "${initdir:?}/$target" ]] && return 0
2882     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2883
2884     local file line
2885     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2886     # DSOs provided by systemd
2887     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2888     local wrap_binary=0
2889     local enable_lsan=0
2890     # I love bash!
2891     while read -r line; do
2892         [[ "$line" = 'not a dynamic executable' ]] && break
2893
2894         # Ignore errors about our own stuff missing. This is most likely caused
2895         # by ldd attempting to use the unprefixed RPATH.
2896         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2897
2898         # We're built with ASan and the target binary loads one of the systemd's
2899         # DSOs, so we need to tweak the environment before executing the binary
2900         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2901             wrap_binary=1
2902         fi
2903
2904         if [[ "$line" =~ $so_regex ]]; then
2905             file="${BASH_REMATCH[1]}"
2906             [[ -e "${initdir}/$file" ]] && continue
2907             inst_library "$file"
2908             continue
2909         fi
2910
2911         if [[ "$line" =~ not\ found ]]; then
2912             dfatal "Missing a shared library required by $bin."
2913             dfatal "Run \"ldd $bin\" to find out what it is."
2914             dfatal "$line"
2915             dfatal "Cannot create a test image."
2916             exit 1
2917         fi
2918     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2919
2920     # Same as above, but we need to wrap certain libraries unconditionally
2921     #
2922     # chgrp, chown, getent, login, setfacl, su, useradd, userdel
2923     #    - dlopen() (not only) systemd's PAM modules
2924     # ls, mkfs.*, mksquashfs, mkswap, setpriv, stat
2925     #   - pull in nss_systemd with certain options (like ls -l) when
2926     #     nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2927     # delv, dig - pull in nss_resolve if `resolve` is in nsswitch.conf
2928     # tar - called by machinectl in TEST-25
2929     bin_rx='/(agetty|chgrp|chown|curl|delv|dig|getfacl|getent|id|login|ls|mkfs\.[a-z0-9]+|mksquashfs|mkswap|setfacl|setpriv|stat|su|tar|useradd|userdel)$'
2930     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ $bin_rx ]]; then
2931         wrap_binary=1
2932         # Ugh, so we want to disable LSan in most cases for the wrapped binaries, since
2933         # we don't care about memory leaks in such binaries. However, in certain cases
2934         # the external binary is the only interface for the systemd code, like for
2935         # the systemd NSS modules, where we want to detect memory leaks. So let's
2936         # do another check to decide if we want to enable LSan for given binary.
2937         if [[ "$bin" =~ /getent$ ]]; then
2938             enable_lsan=1
2939         fi
2940     fi
2941
2942     # If the target binary is built with ASan support, we don't need to wrap
2943     # it, as it should handle everything by itself
2944     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2945         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2946         # Install the target binary with a ".orig" suffix
2947         inst_simple "$bin" "${target}.orig"
2948         # Create a simple shell wrapper in place of the target binary, which
2949         # sets necessary ASan-related env variables and then exec()s the
2950         # suffixed target binary
2951         cat >"$initdir/$target" <<EOF
2952 #!/bin/bash
2953 # Preload the ASan runtime DSO, otherwise ASAn will complain
2954 export LD_PRELOAD="$ASAN_RT_PATH"
2955 # Disable LSan to speed things up, since we don't care about leak reports
2956 # from 'external' binaries
2957 export ASAN_OPTIONS=detect_leaks=$enable_lsan
2958 # Set argv[0] to the original binary name without the ".orig" suffix
2959 exec -a "\$0" -- "${target}.orig" "\$@"
2960 EOF
2961         chmod +x "$initdir/$target"
2962     else
2963         inst_simple "$bin" "$target"
2964     fi
2965 }
2966
2967 # same as above, except for shell scripts.
2968 # If your shell script does not start with shebang, it is not a shell script.
2969 inst_script() {
2970     local bin line shebang_regex
2971     bin="$(find_binary "${1:?}")" || return 1
2972     shift
2973
2974     read -r -n 80 line <"$bin"
2975     # If debug is set, clean unprintable chars to prevent messing up the term
2976     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2977     shebang_regex='(#! *)(/[^ ]+).*'
2978     [[ "$line" =~ $shebang_regex ]] || return 1
2979     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2980 }
2981
2982 # same as above, but specialized for symlinks
2983 inst_symlink() {
2984     local src="${1:?}"
2985     local target="${2:-$src}"
2986     local realsrc
2987
2988     strstr "$src" "/" || return 1
2989     [[ -L "$src" ]] || return 1
2990     [[ -L "${initdir:?}/$target" ]] && return 0
2991     realsrc="$(readlink -f "$src")"
2992     if ! [[ -e "$initdir/$realsrc" ]]; then
2993         if [[ -d "$realsrc" ]]; then
2994             inst_dir "$realsrc"
2995         else
2996             inst "$realsrc"
2997         fi
2998     fi
2999     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
3000     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
3001     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
3002 }
3003
3004 # attempt to install any programs specified in a udev rule
3005 inst_rule_programs() {
3006     local rule="${1:?}"
3007     local prog bin
3008
3009     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
3010         if [ -x "/lib/udev/$prog" ]; then
3011             bin="/lib/udev/$prog"
3012         else
3013             if ! bin="$(find_binary "$prog")"; then
3014                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
3015                 continue
3016             fi
3017         fi
3018
3019         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
3020         image_install "$bin"
3021     done
3022 }
3023
3024 # udev rules always get installed in the same place, so
3025 # create a function to install them to make life simpler.
3026 inst_rules() {
3027     local target=/etc/udev/rules.d
3028     local found rule
3029
3030     inst_dir "/lib/udev/rules.d"
3031     inst_dir "$target"
3032     for rule in "$@"; do
3033         if [ "${rule#/}" = "$rule" ]; then
3034             for r in /lib/udev/rules.d /etc/udev/rules.d; do
3035                 if [[ -f "$r/$rule" ]]; then
3036                     found="$r/$rule"
3037                     inst_simple "$found"
3038                     inst_rule_programs "$found"
3039                 fi
3040             done
3041         fi
3042         for r in '' ./; do
3043             if [[ -f "${r}${rule}" ]]; then
3044                 found="${r}${rule}"
3045                 inst_simple "$found" "$target/${found##*/}"
3046                 inst_rule_programs "$found"
3047             fi
3048         done
3049         [[ $found ]] || dinfo "Skipping udev rule: $rule"
3050         found=
3051     done
3052 }
3053
3054 # general purpose installation function
3055 # Same args as above.
3056 inst() {
3057     case $# in
3058         1) ;;
3059         2)
3060             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
3061             [[ "$initdir" = "$2" ]] && set "$1"
3062             ;;
3063         3)
3064             [[ -z "$initdir" ]] && export initdir="$2"
3065             set "$1" "$3"
3066             ;;
3067         *)
3068             dfatal "inst only takes 1 or 2 or 3 arguments"
3069             exit 1
3070             ;;
3071     esac
3072
3073     local fun
3074     for fun in inst_symlink inst_script inst_binary inst_simple; do
3075         "$fun" "$@" && return 0
3076     done
3077
3078     dwarn "Failed to install '$1'"
3079     return 1
3080 }
3081
3082 # install any of listed files
3083 #
3084 # If first argument is '-d' and second some destination path, first accessible
3085 # source is installed into this path, otherwise it will installed in the same
3086 # path as source.  If none of listed files was installed, function return 1.
3087 # On first successful installation it returns with 0 status.
3088 #
3089 # Example:
3090 #
3091 # inst_any -d /bin/foo /bin/bar /bin/baz
3092 #
3093 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
3094 # initrd.
3095 inst_any() {
3096     local dest file
3097
3098     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
3099
3100     for file in "$@"; do
3101         if [[ -e "$file" ]]; then
3102             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
3103             inst "$file" && return 0
3104         fi
3105     done
3106
3107     return 1
3108 }
3109
3110 inst_recursive() {
3111     local p item
3112
3113     for p in "$@"; do
3114         # Make sure the source exists, as the process substitution below
3115         # suppresses errors
3116         stat "$p" >/dev/null || return 1
3117
3118         while read -r item; do
3119             if [[ -d "$item" ]]; then
3120                 inst_dir "$item"
3121             elif [[ -f "$item" ]]; then
3122                 inst_simple "$item"
3123             fi
3124         done < <(find "$p" 2>/dev/null)
3125     done
3126 }
3127
3128 # image_install [-o ] <file> [<file> ... ]
3129 # Install <file> to the test image
3130 # -o optionally install the <file> and don't fail, if it is not there
3131 image_install() {
3132     local optional=no
3133     local prog="${1:?}"
3134
3135     if [[ "$prog" = '-o' ]]; then
3136         optional=yes
3137         shift
3138     fi
3139
3140     for prog in "$@"; do
3141         if ! inst "$prog" ; then
3142             if get_bool "$optional"; then
3143                 dinfo "Skipping program $prog as it cannot be found and is" \
3144                     "flagged to be optional"
3145             else
3146                 dfatal "Failed to install $prog"
3147                 exit 1
3148             fi
3149         fi
3150     done
3151 }
3152
3153 # Install a single kernel module along with any firmware it may require.
3154 # $1 = full path to kernel module to install
3155 install_kmod_with_fw() {
3156     local module="${1:?}"
3157     # no need to go further if the module is already installed
3158     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
3159     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
3160
3161     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
3162
3163     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
3164
3165     local modname="${module##*/}"
3166     local fwdir found fw
3167     modname="${modname%.ko*}"
3168
3169     while read -r fw; do
3170         found=
3171         for fwdir in /lib/firmware/updates /lib/firmware; do
3172             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
3173                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
3174                 found=yes
3175             fi
3176         done
3177         if ! get_bool "$found"; then
3178             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
3179                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
3180                     "\"${modname}.ko\""
3181             else
3182                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
3183                     "\"${modname}.ko\""
3184             fi
3185         fi
3186     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
3187     return 0
3188 }
3189
3190 # Do something with all the dependencies of a kernel module.
3191 # Note that kernel modules depend on themselves using the technique we use
3192 # $1 = function to call for each dependency we find
3193 #      It will be passed the full path to the found kernel module
3194 # $2 = module to get dependencies for
3195 # rest of args = arguments to modprobe
3196 for_each_kmod_dep() {
3197     local func="${1:?}"
3198     local kmod="${2:?}"
3199     local found=0
3200     local cmd modpath
3201     shift 2
3202
3203     while read -r cmd modpath _; do
3204         [[ "$cmd" = insmod ]] || continue
3205         "$func" "$modpath" || return $?
3206         found=1
3207     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
3208
3209     ! get_bool "$found" && return 1
3210     return 0
3211 }
3212
3213 # instmods [-c] <kernel module> [<kernel module> ... ]
3214 # instmods [-c] <kernel subsystem>
3215 # install kernel modules along with all their dependencies.
3216 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
3217 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
3218 instmods() {
3219     local check=no
3220     if [[ $# -ge 0 && "$1" = '-c' ]]; then
3221         check=yes
3222         shift
3223     fi
3224
3225     inst1mod() {
3226         local mod="${1:?}"
3227         local ret=0
3228         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
3229
3230         case "$mod" in
3231             =*)
3232                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
3233                     (
3234                         [[ "$mpargs" ]] && echo "$mpargs"
3235                         cat "${mod_dir}/modules.${mod#=}"
3236                     ) | instmods
3237                 else
3238                     (
3239                         [[ "$mpargs" ]] && echo "$mpargs"
3240                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
3241                     ) | instmods
3242                 fi
3243                 ;;
3244             --*)
3245                 mpargs+=" $mod"
3246                 ;;
3247             i2o_scsi)
3248                 # Do not load this diagnostic-only module
3249                 return
3250                 ;;
3251             *)
3252                 mod=${mod##*/}
3253                 # if we are already installed, skip this module and go on
3254                 # to the next one.
3255                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
3256
3257                 # We use '-d' option in modprobe only if modules prefix path
3258                 # differs from default '/'.  This allows us to use Dracut with
3259                 # old version of modprobe which doesn't have '-d' option.
3260                 local mod_dirname=${mod_dir%%/lib/modules/*}
3261                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
3262
3263                 # ok, load the module, all its dependencies, and any firmware
3264                 # it may require
3265                 for_each_kmod_dep install_kmod_with_fw "$mod" \
3266                     --set-version "$KERNEL_VER" \
3267                     ${mod_dirname:+"$mod_dirname"} \
3268                     ${mpargs:+"$mpargs"}
3269                 ((ret+=$?))
3270                 ;;
3271         esac
3272         return "$ret"
3273     }
3274
3275     local mod mpargs
3276
3277     if [[ $# -eq 0 ]]; then  # filenames from stdin
3278         while read -r mod; do
3279             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3280                 dfatal "Failed to install $mod"
3281                 return 1
3282             fi
3283         done
3284     fi
3285
3286     for mod in "$@"; do # filenames as arguments
3287         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3288             dfatal "Failed to install $mod"
3289             return 1
3290         fi
3291     done
3292
3293     return 0
3294 }
3295
3296 _umount_dir() {
3297     local mountpoint="${1:?}"
3298     if mountpoint -q "$mountpoint"; then
3299         ddebug "umount $mountpoint"
3300         umount "$mountpoint"
3301     fi
3302 }
3303
3304 # can be overridden in specific test
3305 test_setup_cleanup() {
3306     cleanup_initdir
3307 }
3308
3309 _test_cleanup() {
3310     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3311     (
3312         set +e
3313         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3314         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3315         # If multiple setups/cleans are ran in parallel, this can cause a race
3316         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3317             rm -vf "${IMAGESTATEDIR}/default.img"
3318         fi
3319         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3320         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3321     ) || :
3322 }
3323
3324 # can be overridden in specific test
3325 test_cleanup() {
3326     _test_cleanup
3327 }
3328
3329 test_cleanup_again() {
3330     [ -n "$TESTDIR" ] || return
3331     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3332     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3333 }
3334
3335 test_create_image() {
3336     create_empty_image_rootdir
3337
3338     # Create what will eventually be our root filesystem onto an overlay
3339     (
3340         LOG_LEVEL=5
3341         setup_basic_environment
3342     )
3343 }
3344
3345 test_setup() {
3346     if ! get_bool "$NO_BUILD" && \
3347          get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3348          command -v meson >/dev/null && \
3349          [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3350         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3351         exit 1
3352     fi
3353
3354     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3355         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3356         mount_initdir
3357     else
3358         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3359             # default.img is the base that every test uses and optionally appends to
3360             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3361                 # Create the backing public image, but then completely unmount
3362                 # it and drop the loopback device responsible for it, since we're
3363                 # going to symlink/copy the image and mount it again from
3364                 # elsewhere.
3365                 local image_old="${IMAGE_PUBLIC}"
3366                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3367                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3368                 fi
3369                 test_create_image
3370                 test_setup_cleanup
3371                 umount_loopback
3372                 cleanup_loopdev
3373                 IMAGE_PUBLIC="${image_old}"
3374             fi
3375             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3376                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3377             fi
3378         fi
3379
3380         local hook_defined
3381         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3382
3383         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3384         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3385             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3386         else
3387             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3388         fi
3389
3390         mount_initdir
3391
3392         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3393             dinfo "Masking supporting services"
3394             mask_supporting_services
3395         fi
3396
3397         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
3398             # Do an initial coverage capture, to make sure the final report includes
3399             # files that the tests didn't touch at all
3400             lcov --initial --capture --directory "${initdir}/${BUILD_DIR:?}" --output-file "${TESTDIR:?}/coverage-base"
3401         fi
3402
3403         if get_bool "$hook_defined"; then
3404             test_append_files "${initdir:?}"
3405         fi
3406     fi
3407
3408     setup_nspawn_root
3409 }
3410
3411 test_run() {
3412     local test_id="${1:?}"
3413     mount_initdir
3414
3415     if ! get_bool "${TEST_NO_QEMU:=}"; then
3416         if run_qemu "$test_id"; then
3417             check_result_qemu || { echo "qemu test failed"; return 1; }
3418         else
3419             dwarn "can't run qemu, skipping"
3420         fi
3421     fi
3422     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3423         mount_initdir
3424         if run_nspawn "${initdir:?}" "$test_id"; then
3425             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3426         else
3427             dwarn "can't run systemd-nspawn, skipping"
3428         fi
3429
3430         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3431             dir="$TESTDIR/unprivileged-nspawn-root"
3432             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3433                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3434             else
3435                 dwarn "can't run systemd-nspawn, skipping"
3436             fi
3437         fi
3438     fi
3439     return 0
3440 }
3441
3442 do_test() {
3443     if [[ $UID != "0" ]]; then
3444         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3445         exit 0
3446     fi
3447
3448     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3449         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3450         exit 0
3451     fi
3452
3453     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3454         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3455         exit 0
3456     fi
3457
3458     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3459         TEST_NO_QEMU=1
3460     fi
3461
3462     # Detect lib paths
3463     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3464         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3465     done
3466
3467     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3468         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3469     done
3470
3471     mkdir -p "$WORKDIR"
3472     mkdir -p "$STATEDIR"
3473
3474     import_testdir
3475     import_initdir
3476
3477     if [ -n "${SUDO_USER}" ]; then
3478         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3479         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3480     fi
3481
3482     testname="$(basename "$PWD")"
3483
3484     while (($# > 0)); do
3485         case $1 in
3486             --run)
3487                 echo "${testname} RUN: $TEST_DESCRIPTION"
3488                 test_run "$TESTID"
3489                 ret=$?
3490                 if [ $ret -eq 0 ]; then
3491                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3492                 else
3493                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3494                 fi
3495                 exit $ret
3496                 ;;
3497             --setup)
3498                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3499                 test_setup
3500                 test_setup_cleanup
3501                 ;;
3502             --clean)
3503                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3504                 test_cleanup
3505                 ;;
3506             --clean-again)
3507                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3508                 test_cleanup_again
3509                 ;;
3510             --all)
3511                 ret=0
3512                 echo -n "${testname}: $TEST_DESCRIPTION "
3513                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3514                 # and loop devices will leak
3515                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3516                 if [ $ret -eq 0 ]; then
3517                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3518                 fi
3519                 if [ $ret -eq 0 ]; then
3520                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3521                 fi
3522                 test_cleanup
3523                 if [ $ret -eq 0 ]; then
3524                     # $TESTLOG is in $STATEDIR, so clean it up only on success
3525                     [[ -n "$STATEDIR" ]] && rm -vfr "$STATEDIR"
3526                     echo "[OK]"
3527                 else
3528                     echo "[FAILED]"
3529                     echo "see $TESTLOG"
3530                 fi
3531                 exit $ret
3532                 ;;
3533             *)
3534                 break
3535                 ;;
3536         esac
3537         shift
3538     done
3539 }