test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  18 export PATH
  19
  20 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  21 # shellcheck source=/dev/null
  22 source "$os_release"
  23 [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=""
  24 [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=""
  25 [[ " $ID_LIKE " = *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=""
  26 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  27 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  28 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  29 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  30 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  31 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  32 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  33 # Note that defining a different IMAGE_NAME in a test setup script will only result
  34 # in default.img being copied and renamed. It can then be extended by defining
  35 # a test_append_files() function. The $1 parameter will be the root directory.
  36 # To force creating a new image from scratch (eg: to encrypt it), also define
  37 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  38 IMAGE_NAME=${IMAGE_NAME:-default}
  39 STRIP_BINARIES="${STRIP_BINARIES:-yes}"
  40 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  41 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  42 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  43 LOOPDEV=
  44
  45 # Simple wrapper to unify boolean checks.
  46 # Note: this function needs to stay near the top of the file, so we can use it
  47 #       in code in the outermost scope.
  48 get_bool() {
  49     # Make the value lowercase to make the regex matching simpler
  50     local _bool="${1,,}"
  51
  52     # Consider empty value as "false"
  53     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  54         return 1
  55     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  56         return 0
  57     else
  58         echo >&2 "Value '$_bool' is not a valid boolean value"
  59         exit 1
  60     fi
  61 }
  62
  63 # Since in Bash we can have only one handler per signal, let's overcome this
  64 # limitation by having one global handler for the EXIT signal which executes
  65 # all registered handlers
  66 _AT_EXIT_HANDLERS=()
  67 _at_exit() {
  68     set +e
  69
  70     # Run the EXIT handlers in reverse order
  71     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  72         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  73         "${_AT_EXIT_HANDLERS[$i]}"
  74     done
  75 }
  76
  77 trap _at_exit EXIT
  78
  79 add_at_exit_handler() {
  80     local handler="${1?}"
  81
  82     if [[ "$(type -t "$handler")" != "function" ]]; then
  83         dfatal "'$handler' is not a function"
  84         exit 1
  85     fi
  86
  87     _AT_EXIT_HANDLERS+=("$handler")
  88 }
  89
  90 # Decide if we can (and want to) run qemu with KVM acceleration.
  91 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  92 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  93 # running under KVM. If these conditions are met, enable KVM (and possibly
  94 # nested KVM), otherwise disable it.
  95 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  96     QEMU_KVM=yes
  97 else
  98     QEMU_KVM=no
  99 fi
 100
 101 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
 102     echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
 103     ROOTLIBDIR=/usr/lib/systemd
 104 fi
 105
 106 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
 107 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 108 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 109 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 110 TOOLS_DIR="$SOURCE_DIR/tools"
 111 # These variables are used by test scripts
 112 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR TOOLS_DIR
 113
 114 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 115 if get_bool "${NO_BUILD:=}"; then
 116     BUILD_DIR="$SOURCE_DIR"
 117 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 118     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 119     exit 1
 120 fi
 121
 122 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 123 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 124 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 125 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 126 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 127 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 128 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 129
 130 TESTFILE="${BASH_SOURCE[1]}"
 131 if [ -z "$TESTFILE" ]; then
 132     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 133     exit 1
 134 fi
 135 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 136 STATEDIR="$BUILD_DIR/test/$TESTNAME"
 137 STATEFILE="$STATEDIR/.testdir"
 138 IMAGESTATEDIR="$STATEDIR/.."
 139 TESTLOG="$STATEDIR/test.log"
 140
 141 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 142     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 143     exit 1
 144 fi
 145 TESTID="${BASH_REMATCH[1]:?}"
 146
 147 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 148     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 149     exit 1
 150 fi
 151
 152 BASICTOOLS=(
 153     awk
 154     base64
 155     basename
 156     bash
 157     capsh
 158     cat
 159     chmod
 160     chown
 161     chroot
 162     cmp
 163     cryptsetup
 164     cut
 165     date
 166     dd
 167     diff
 168     dirname
 169     dmsetup
 170     echo
 171     env
 172     false
 173     flock
 174     getconf
 175     getent
 176     getfacl
 177     grep
 178     gunzip
 179     gzip
 180     head
 181     ionice
 182     ip
 183     jq
 184     killall
 185     ldd
 186     ln
 187     loadkeys
 188     login
 189     losetup
 190     lz4cat
 191     mkfifo
 192     mktemp
 193     modprobe
 194     mount
 195     mountpoint
 196     mv
 197     nc
 198     nproc
 199     pkill
 200     readlink
 201     realpath
 202     rev
 203     rm
 204     rmdir
 205     rmmod
 206     sed
 207     seq
 208     setfattr
 209     setfont
 210     setsid
 211     sfdisk
 212     sh
 213     sleep
 214     stat
 215     su
 216     sulogin
 217     sysctl
 218     tail
 219     tar
 220     tee
 221     test
 222     timeout
 223     touch
 224     tr
 225     true
 226     truncate
 227     umount
 228     uname
 229     unshare
 230     useradd
 231     userdel
 232     wc
 233     xargs
 234     xzcat
 235 )
 236
 237 DEBUGTOOLS=(
 238     cp
 239     df
 240     dhclient
 241     dmesg
 242     du
 243     find
 244     free
 245     grep
 246     hostname
 247     id
 248     less
 249     ln
 250     ls
 251     mkdir
 252     ping
 253     ps
 254     route
 255     sort
 256     strace
 257     stty
 258     tty
 259     vi
 260     /usr/libexec/vi
 261 )
 262
 263 is_built_with_asan() {
 264     local _bin="${1:?}"
 265
 266     if ! type -P objdump >/dev/null; then
 267         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
 268         return 1
 269     fi
 270
 271     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 272     local _asan_calls
 273     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 274     if ((_asan_calls < 1000)); then
 275         return 1
 276     else
 277         return 0
 278     fi
 279 }
 280
 281 is_built_with_coverage() {
 282     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 283         return 1
 284     fi
 285
 286     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 287 }
 288
 289 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 290 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 291
 292 if get_bool "$IS_BUILT_WITH_ASAN"; then
 293     STRIP_BINARIES=no
 294     SKIP_INITRD="${SKIP_INITRD:-yes}"
 295     PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
 296     QEMU_MEM="${QEMU_MEM:-2G}"
 297     QEMU_SMP="${QEMU_SMP:-4}"
 298
 299     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 300     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 301         ASAN_COMPILER=gcc
 302         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 303     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 304         ASAN_COMPILER=clang
 305         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 306
 307         # As clang's ASan DSO is usually in a non-standard path, let's check if
 308         # the environment is set accordingly. If not, warn the user and exit.
 309         # We're not setting the LD_LIBRARY_PATH automagically here, because
 310         # user should encounter (and fix) the same issue when running the unit
 311         # tests (meson test)
 312         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 313             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 314             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 315             exit 1
 316         fi
 317     else
 318         echo >&2 "systemd is not linked against the ASan DSO"
 319         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 320         exit 1
 321     fi
 322
 323     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 324 fi
 325
 326 find_qemu_bin() {
 327     QEMU_BIN="${QEMU_BIN:-""}"
 328     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 329     if get_bool "$QEMU_KVM"; then
 330         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 331     fi
 332
 333     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 334     case $ARCH in
 335     x86_64)
 336         # QEMU's own build system calls it qemu-system-x86_64
 337         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 338         ;;
 339     i*86)
 340         # new i386 version of QEMU
 341         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 342
 343         # i386 version of QEMU
 344         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 345         ;;
 346     ppc64*)
 347         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 348         ;;
 349     esac
 350
 351     if [[ ! -e "$QEMU_BIN" ]]; then
 352         echo "Could not find a suitable qemu binary" >&2
 353         return 1
 354     fi
 355 }
 356
 357 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 358 # returns 0 if newer or equal
 359 # returns 1 if older
 360 # returns 2 if failing
 361 qemu_min_version() {
 362     find_qemu_bin || return 2
 363
 364     # get version from binary
 365     local qemu_ver
 366     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 367
 368     # Check version string format
 369     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 370     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 371
 372     # compare as last command to return that value
 373     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 374 }
 375
 376 # Return 0 if qemu did run (then you must check the result state/logs for actual
 377 # success), or 1 if qemu is not available.
 378 run_qemu() {
 379     # If the test provided its own initrd, use it (e.g. TEST-24)
 380     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 381         INITRD="$TESTDIR/initrd.img"
 382     fi
 383
 384     if [ -f /etc/machine-id ]; then
 385         read -r MACHINE_ID </etc/machine-id
 386         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 387             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 388         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 389             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 390     fi
 391
 392     local CONSOLE=ttyS0
 393
 394     rm -f "$initdir"/{testok,failed,skipped}
 395     # make sure the initdir is not mounted to avoid concurrent access
 396     cleanup_initdir
 397     umount_loopback
 398
 399     if [[ ! "$KERNEL_BIN" ]]; then
 400         if get_bool "$LOOKS_LIKE_ARCH"; then
 401             KERNEL_BIN=/boot/vmlinuz-linux
 402         else
 403             [ "$ARCH" ] || ARCH=$(uname -m)
 404             case $ARCH in
 405                 ppc64*)
 406                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 407                     # uses the "standard" vmlinuz- prefix
 408                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 409                     CONSOLE=hvc0
 410                     ;;
 411                 *)
 412                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 413                     ;;
 414             esac
 415         fi
 416     fi
 417
 418     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 419     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 420     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 421     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 422     if [[ ! "$INITRD" ]]; then
 423         if [[ -e "$default_fedora_initrd" ]]; then
 424             INITRD="$default_fedora_initrd"
 425         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 426             INITRD="$default_debian_initrd"
 427         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 428             INITRD="$default_arch_initrd"
 429         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 430             INITRD="$default_suse_initrd"
 431         fi
 432     fi
 433
 434     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 435     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 436     # is not installed or there's some other error when calling it, fall back
 437     # to the original value (QEMU_SMP=1).
 438     if [[ -z "${QEMU_SMP:=}" ]]; then
 439         if ! QEMU_SMP=$(nproc); then
 440             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 441             QEMU_SMP=1
 442         fi
 443     fi
 444
 445     find_qemu_bin || return 1
 446
 447     # Umount initdir to avoid concurrent access to the filesystem
 448     _umount_dir "$initdir"
 449
 450     local kernel_params=()
 451     local qemu_options=()
 452     local qemu_cmd=("$QEMU_BIN")
 453
 454     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 455         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 456     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 457         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 458     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 459         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 460     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 461         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 462         exit 1
 463     fi
 464
 465     if get_bool "$LOOKS_LIKE_SUSE"; then
 466         kernel_params+=("rd.hostonly=0")
 467     fi
 468
 469     kernel_params+=(
 470         "root=LABEL=systemd_boot"
 471         "rw"
 472         "raid=noautodetect"
 473         "rd.luks=0"
 474         "loglevel=2"
 475         "init=$PATH_TO_INIT"
 476         "console=$CONSOLE"
 477         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 478         "systemd.unit=testsuite.target"
 479         "systemd.wants=testsuite-$1.service"
 480     )
 481
 482     if ! get_bool "$INTERACTIVE_DEBUG"; then
 483         kernel_params+=(
 484             "oops=panic"
 485             "panic=1"
 486             "softlockup_panic=1"
 487             "systemd.wants=end.service"
 488         )
 489     fi
 490
 491     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 492     qemu_options+=(
 493         -smp "$QEMU_SMP"
 494         -net none
 495         -m "${QEMU_MEM:-768M}"
 496         -nographic
 497         -kernel "$KERNEL_BIN"
 498         -drive "format=raw,cache=unsafe,file=$image"
 499         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 500     )
 501
 502     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 503         local user_qemu_options
 504         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 505         qemu_options+=("${user_qemu_options[@]}")
 506     fi
 507
 508     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 509         local user_kernel_append
 510         readarray user_kernel_append <<< "$KERNEL_APPEND"
 511         kernel_params+=("${user_kernel_append[@]}")
 512     fi
 513
 514     if [[ "$INITRD" ]] && ! get_bool "$SKIP_INITRD"; then
 515         qemu_options+=(-initrd "$INITRD")
 516     fi
 517
 518     # Let's use KVM if possible
 519     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 520         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 521     fi
 522
 523     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 524         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 525     fi
 526
 527     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]}")
 528     rc=$?
 529     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 530         derror "Test timed out after ${QEMU_TIMEOUT}s"
 531         TIMED_OUT=1
 532     else
 533         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 534     fi
 535     return 0
 536 }
 537
 538 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 539 # success), or 1 if nspawn is not available.
 540 run_nspawn() {
 541     [[ -d /run/systemd/system ]] || return 1
 542     rm -f "${initdir:?}"/{testok,failed,skipped}
 543
 544     local nspawn_cmd=()
 545     local nspawn_options=(
 546         "--register=no"
 547         "--kill-signal=SIGKILL"
 548         "--directory=${1:?}"
 549         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 550         "--machine=TEST-$TESTID"
 551     )
 552     local kernel_params=(
 553         "$PATH_TO_INIT"
 554         "systemd.unit=testsuite.target"
 555         "systemd.wants=testsuite-$2.service"
 556     )
 557
 558     if ! get_bool "$INTERACTIVE_DEBUG"; then
 559         kernel_params+=("systemd.wants=end.service")
 560     fi
 561
 562     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 563         local user_nspawn_arguments
 564         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 565         nspawn_options+=("${user_nspawn_arguments[@]}")
 566     fi
 567
 568     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 569         local user_kernel_append
 570         readarray user_kernel_append <<< "$KERNEL_APPEND"
 571         kernel_params+=("${user_kernel_append[@]}")
 572     fi
 573
 574     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 575         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 576         exit
 577     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 578         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 579     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 580         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 581     else
 582         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 583         exit 1
 584     fi
 585
 586     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 587         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 588     else
 589         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 590     fi
 591
 592     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}")
 593     rc=$?
 594     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 595         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 596         TIMED_OUT=1
 597     else
 598         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 599     fi
 600     return 0
 601 }
 602
 603 # Build two very minimal root images, with two units, one is the same and one is different across them
 604 install_verity_minimal() {
 605     dinfo "Set up a set of minimal images for verity verification"
 606     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 607         return
 608     fi
 609     if ! command -v mksquashfs >/dev/null 2>&1; then
 610         dfatal "mksquashfs not found"
 611         exit 1
 612     fi
 613     if ! command -v veritysetup >/dev/null 2>&1; then
 614         dfatal "veritysetup not found"
 615         exit 1
 616     fi
 617     # Local modifications of some global variables is intentional in this
 618     # subshell (SC2030)
 619     # shellcheck disable=SC2030
 620     (
 621         BASICTOOLS=(
 622             bash
 623             cat
 624             grep
 625             mount
 626             sleep
 627         )
 628         oldinitdir="$initdir"
 629         rm -rfv "$TESTDIR/minimal"
 630         export initdir="$TESTDIR/minimal"
 631         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 632         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 633         setup_basic_dirs
 634         install_basic_tools
 635         install_ld_so_conf
 636         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 637         # issue, thus falsely triggering SC2030 in this case
 638         # See: koalaman/shellcheck#1409
 639         if [[ -v ASAN_RT_PATH ]]; then
 640             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 641             # into the verity images to get rid of the annoying errors about
 642             # missing $LD_PRELOAD libraries.
 643             inst_libs "$ASAN_RT_PATH"
 644             inst_library "$ASAN_RT_PATH"
 645         fi
 646         cp "$os_release" "$initdir/usr/lib/os-release"
 647         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 648         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 649         touch "$initdir/opt/some_file"
 650         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 651         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 652         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 653 [Service]
 654 ExecStartPre=cat /usr/lib/os-release
 655 ExecStart=sleep 120
 656 EOF
 657         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 658
 659         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 660         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 661             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 662
 663         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 664         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 665         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 666
 667         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 668         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 669             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 670
 671         # Rolling distros like Arch do not set VERSION_ID
 672         local version_id=""
 673         if grep -q "^VERSION_ID=" "$os_release"; then
 674             version_id="$(grep "^VERSION_ID=" "$os_release")"
 675         fi
 676
 677         export initdir="$TESTDIR/app0"
 678         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 679         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 680         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 681         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 682 [Service]
 683 Type=oneshot
 684 RemainAfterExit=yes
 685 ExecStart=/opt/script0.sh
 686 TemporaryFileSystem=/var/lib
 687 StateDirectory=app0
 688 RuntimeDirectory=app0
 689 EOF
 690         cat >"$initdir/opt/script0.sh" <<EOF
 691 #!/bin/bash
 692 set -e
 693 test -e /usr/lib/os-release
 694 echo bar > \${STATE_DIRECTORY}/foo
 695 cat /usr/lib/extension-release.d/extension-release.app0
 696 EOF
 697         chmod +x "$initdir/opt/script0.sh"
 698         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 699         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 700
 701         export initdir="$TESTDIR/app1"
 702         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 703         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 704         ( echo "${version_id}"
 705           echo "SYSEXT_SCOPE=portable"
 706           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 707         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 708         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 709 [Service]
 710 Type=oneshot
 711 RemainAfterExit=yes
 712 ExecStart=/opt/script1.sh
 713 StateDirectory=app1
 714 RuntimeDirectory=app1
 715 EOF
 716         cat >"$initdir/opt/script1.sh" <<EOF
 717 #!/bin/bash
 718 set -e
 719 test -e /usr/lib/os-release
 720 echo baz > \${STATE_DIRECTORY}/foo
 721 cat /usr/lib/extension-release.d/extension-release.app2
 722 EOF
 723         chmod +x "$initdir/opt/script1.sh"
 724         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 725         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 726
 727         export initdir="$TESTDIR/app-nodistro"
 728         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 729         ( echo "ID=_any"
 730           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 731         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 732         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 733     )
 734 }
 735
 736 setup_basic_environment() {
 737     # create the basic filesystem layout
 738     setup_basic_dirs
 739
 740     install_systemd
 741     install_missing_libraries
 742     install_config_files
 743     install_zoneinfo
 744     create_rc_local
 745     install_basic_tools
 746     install_libnss
 747     install_pam
 748     install_dbus
 749     install_fonts
 750     install_locales
 751     install_keymaps
 752     install_x11_keymaps
 753     install_terminfo
 754     install_execs
 755     install_fs_tools
 756     install_modules
 757     install_plymouth
 758     install_haveged
 759     install_debug_tools
 760     install_ld_so_conf
 761     install_testuser
 762     has_user_dbus_socket && install_user_dbus
 763     setup_selinux
 764     strip_binaries
 765     instmods veth
 766     install_depmod_files
 767     generate_module_dependencies
 768     if get_bool "$IS_BUILT_WITH_ASAN"; then
 769         create_asan_wrapper
 770     fi
 771     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 772         install_verity_minimal
 773     fi
 774 }
 775
 776 setup_selinux() {
 777     dinfo "Setup SELinux"
 778     # don't forget KERNEL_APPEND='... selinux=1 ...'
 779     if ! get_bool "$SETUP_SELINUX"; then
 780         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 781         return 0
 782     fi
 783
 784     local conf_dir=/etc/selinux
 785     local fixfiles_tools=(bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles)
 786
 787     # Make sure the following statement can't expand to "/" to prevent
 788     # a potential where-are-my-backups situation
 789     rm -rf "${initdir:?}/$conf_dir"
 790     if ! cp -ar "$conf_dir" "$initdir/$conf_dir"; then
 791         dfatal "Failed to copy $conf_dir"
 792         exit 1
 793     fi
 794
 795     touch "$initdir/.autorelabel"
 796     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 797     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 798
 799     image_install "${fixfiles_tools[@]}"
 800     image_install fixfiles
 801     image_install sestatus
 802 }
 803
 804 install_valgrind() {
 805     if ! type -p valgrind; then
 806         dfatal "Failed to install valgrind"
 807         exit 1
 808     fi
 809
 810     local valgrind_bins valgrind_libs valgrind_dbg_and_supp
 811
 812     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if /^execve\("([^"]+)"/')
 813     image_install "${valgrind_bins[@]}"
 814
 815     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null | perl -lne 'print $1 if m{calling init: (/.*vgpreload_.*)}')
 816     image_install "${valgrind_libs[@]}"
 817
 818     readarray -t valgrind_dbg_and_supp < <(
 819         strace -e open valgrind /bin/true 2>&1 >/dev/null |
 820         perl -lne 'if (my ($fname) = /^open\("([^"]+).*= (?!-)\d+/) { print $fname if $fname =~ /debug|\.supp$/ }'
 821     )
 822     image_install "${valgrind_dbg_and_supp[@]}"
 823 }
 824
 825 create_valgrind_wrapper() {
 826     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 827     ddebug "Create $valgrind_wrapper"
 828     cat >"$valgrind_wrapper" <<EOF
 829 #!/usr/bin/env bash
 830
 831 mount -t proc proc /proc
 832 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 833 EOF
 834     chmod 0755 "$valgrind_wrapper"
 835 }
 836
 837 create_asan_wrapper() {
 838     local asan_wrapper="$initdir/$ROOTLIBDIR/systemd-under-asan"
 839     dinfo "Create ASan wrapper as '$asan_wrapper'"
 840
 841     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 842
 843     # clang: install llvm-symbolizer to generate useful reports
 844     # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
 845     [[ "$ASAN_COMPILER" == "clang" ]] && image_install "llvm-symbolizer"
 846
 847     cat >"$asan_wrapper" <<EOF
 848 #!/usr/bin/env bash
 849
 850 set -x
 851
 852 echo "ASan RT: $ASAN_RT_PATH"
 853 if [[ ! -e "$ASAN_RT_PATH" ]]; then
 854     echo >&2 "Couldn't find ASan RT at '$ASAN_RT_PATH', can't continue"
 855     exit 1
 856 fi
 857
 858 DEFAULT_ASAN_OPTIONS=${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}
 859 DEFAULT_UBSAN_OPTIONS=${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}
 860 DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS"
 861
 862 # Create a simple environment file which can be included by systemd services
 863 # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
 864 cat >/usr/lib/systemd/systemd-asan-env <<INNER_EOF
 865 LD_PRELOAD=$ASAN_RT_PATH
 866 ASAN_OPTIONS=$DEFAULT_ASAN_OPTIONS
 867 LSAN_OPTIONS=detect_leaks=0
 868 UBSAN_OPTIONS=$DEFAULT_UBSAN_OPTIONS
 869 INNER_EOF
 870
 871 # As right now bash is the PID 1, we can't expect PATH to have a sane value.
 872 # Let's make one to prevent unexpected "<bin> not found" issues in the future
 873 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
 874
 875 mount -t proc proc /proc
 876 mount -t sysfs sysfs /sys
 877 mount -o remount,rw /
 878
 879 DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT ASAN_RT_PATH=$ASAN_RT_PATH"
 880
 881 if [[ "$ASAN_COMPILER" == "clang" ]]; then
 882   # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
 883   # unnecessary for gcc & libasan, however, for clang this is crucial, as its
 884   # runtime ASan DSO is in a non-standard (library) path.
 885   echo "${ASAN_RT_PATH%/*}" >/etc/ld.so.conf.d/asan-path-override.conf
 886   ldconfig
 887 fi
 888 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
 889 echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 890 echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf
 891
 892 # ASAN and syscall filters aren't compatible with each other.
 893 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
 894
 895 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
 896 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
 897 JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 898 mkdir -p "\$JOURNALD_CONF_DIR"
 899 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 900
 901 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
 902 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
 903 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
 904 printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"
 905
 906 # 90s isn't enough for some services to finish when literally everything is run
 907 # under ASan+UBSan in containers, which, in turn, are run in VMs.
 908 # Let's limit which environments such services should be executed in.
 909 mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
 910 printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=240s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
 911
 912 # Let's override another hard-coded timeout that kicks in too early
 913 mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
 914 printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf
 915
 916 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 917 exec "$ROOTLIBDIR/systemd" "\$@"
 918 EOF
 919
 920     chmod 0755 "$asan_wrapper"
 921 }
 922
 923 create_strace_wrapper() {
 924     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
 925     ddebug "Create $strace_wrapper"
 926     cat >"$strace_wrapper" <<EOF
 927 #!/usr/bin/env bash
 928
 929 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
 930 EOF
 931     chmod 0755 "$strace_wrapper"
 932 }
 933
 934 install_fs_tools() {
 935     dinfo "Install fsck"
 936     image_install /sbin/fsck*
 937     image_install -o /bin/fsck*
 938
 939     # fskc.reiserfs calls reiserfsck. so, install it
 940     image_install -o reiserfsck
 941
 942     # we use mkfs in system-repart tests
 943     image_install /sbin/mkfs.ext4
 944     image_install /sbin/mkfs.vfat
 945 }
 946
 947 install_modules() {
 948     dinfo "Install modules"
 949
 950     instmods loop
 951     instmods vfat
 952     instmods nls_ascii =nls
 953     instmods dummy
 954     # for TEST-35-LOGIN
 955     instmods scsi_debug uinput
 956
 957     if get_bool "$LOOKS_LIKE_SUSE"; then
 958         instmods ext4
 959         # for TEST-54-CREDS
 960         instmods dmi-sysfs
 961     fi
 962 }
 963
 964 install_dmevent() {
 965     instmods dm_crypt =crypto
 966     inst_binary dmeventd
 967     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
 968     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 969         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
 970         # and since buster/bionic 95-dm-notify.rules
 971         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
 972         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
 973     else
 974         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
 975     fi
 976     if get_bool "$LOOKS_LIKE_SUSE"; then
 977         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
 978     fi
 979 }
 980
 981 install_multipath() {
 982     instmods "=md" multipath
 983     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
 984     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
 985     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 986         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
 987     else
 988         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
 989     fi
 990     mkdir -p "${initdir:?}/etc/multipath"
 991
 992     local file
 993     while read -r file; do
 994         # Install libraries required by the given library
 995         inst_libs "$file"
 996         # Install the library itself and create necessary symlinks
 997         inst_library "$file"
 998     done < <(find /lib*/multipath -type f)
 999 }
1000
1001 install_lvm() {
1002     image_install lvm
1003     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1004     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1005     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1006     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1007         inst_rules 56-lvm.rules 69-lvm-metad.rules
1008     else
1009         # Support the new udev autoactivation introduced in lvm 2.03.14
1010         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1011         # Static autoactivation (via lvm2-activation-generator) was dropped
1012         # in lvm 2.03.15
1013         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1014         if [[ -f /lib/udev/rules.d/69-dm-lvm.rules ]]; then
1015             inst_rules 11-dm-lvm.rules 69-dm-lvm.rules
1016         else
1017             image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1018             image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1019             inst_rules 11-dm-lvm.rules 69-dm-lvm-metad.rules
1020         fi
1021     fi
1022     mkdir -p "${initdir:?}/etc/lvm"
1023 }
1024
1025 install_btrfs() {
1026     instmods btrfs
1027     # Not all utilities provided by btrfs-progs are listed here; extend the list
1028     # if necessary
1029     image_install btrfs btrfstune mkfs.btrfs
1030     inst_rules 64-btrfs-dm.rules
1031 }
1032
1033 install_iscsi() {
1034     # Install both client and server side stuff by default
1035     local inst="${1:-}"
1036     local file
1037
1038     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1039     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1040     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1041         image_install iscsi-iname iscsiadm iscsid iscsistart
1042         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1043         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1044         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1045         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1046         mkdir -p "${initdir:?}/etc/iscsi"
1047         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1048         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1049         # a one-time service instead of distro package's post-install scripts.
1050         # However, some distros still use this approach even after this patch,
1051         # so prefer the already existing initiatorname.iscsi file if it exists.
1052         #
1053         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1054         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1055             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1056             if get_bool "$IS_BUILT_WITH_ASAN"; then
1057                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1058                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1059                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1060                 # (among others) pre-loads ASan's DSO.
1061                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1062                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1063             fi
1064         else
1065             inst_simple "/etc/iscsi/initiatorname.iscsi"
1066         fi
1067     fi
1068
1069     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1070     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1071     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1072         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1073         image_install -o /etc/sysconfig/tgtd
1074         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1075         mkdir -p "${initdir:?}/etc/tgt"
1076         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1077         # Install perl modules required by tgt-admin
1078         #
1079         # Forgive me father for I have sinned. The monstrosity below appends
1080         # a perl snippet to the `tgt-admin` perl script on the fly, which
1081         # dumps a list of files (perl modules) required by `tgt-admin` at
1082         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1083         # passed to `inst_simple` which installs the necessary files into the image
1084         #
1085         # shellcheck disable=SC2016
1086         while read -r file; do
1087             inst_simple "$file"
1088         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1089     fi
1090 }
1091
1092 install_mdadm() {
1093     local unit
1094     local mdadm_units=(
1095         system/mdadm-grow-continue@.service
1096         system/mdadm-last-resort@.service
1097         system/mdadm-last-resort@.timer
1098         system/mdmon@.service
1099         system/mdmonitor-oneshot.service
1100         system/mdmonitor-oneshot.timer
1101         system/mdmonitor.service
1102         system-shutdown/mdadm.shutdown
1103     )
1104
1105     image_install mdadm mdmon
1106     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1107     # Fedora/CentOS/RHEL ships this rule file
1108     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1109
1110     for unit in "${mdadm_units[@]}"; do
1111         image_install "${ROOTLIBDIR:?}/$unit"
1112     done
1113 }
1114
1115 install_compiled_systemd() {
1116     dinfo "Install compiled systemd"
1117
1118     local ninja_bin
1119     ninja_bin="$(type -P ninja || type -P ninja-build)"
1120     if [[ -z "$ninja_bin" ]]; then
1121         dfatal "ninja was not found"
1122         exit 1
1123     fi
1124     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1125
1126     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1127     # find them in the same directory as the runtime data counts
1128     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1129         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1130         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1131         # Set effective & default ACLs for the build dir so unprivileged
1132         # processes can write gcda files with coverage stats
1133         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1134     fi
1135 }
1136
1137 install_debian_systemd() {
1138     dinfo "Install debian systemd"
1139
1140     local files
1141
1142     while read -r deb; do
1143         files="$(dpkg-query -L "$deb" 2>/dev/null)" || continue
1144         ddebug "Install debian files from package $deb"
1145         for file in $files; do
1146             [ -e "$file" ] || continue
1147             [ -d "$file" ] && continue
1148             inst "$file"
1149         done
1150     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1151 }
1152
1153 install_suse_systemd() {
1154     local pkgs
1155
1156     dinfo "Install SUSE systemd"
1157
1158     pkgs=(
1159         systemd
1160         systemd-container
1161         systemd-coredump
1162         systemd-experimental
1163         systemd-journal-remote
1164         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1165         # systemd-network sub-package on openSUSE, has its own testsuite.
1166         systemd-network
1167         systemd-portable
1168         udev
1169     )
1170
1171     for p in "${pkgs[@]}"; do
1172         rpm -q "$p" &>/dev/null || continue
1173
1174         ddebug "Install files from package $p"
1175         while read -r f; do
1176             [ -e "$f" ] || continue
1177             [ -d "$f" ] && continue
1178             inst "$f"
1179         done < <(rpm -ql "$p")
1180     done
1181
1182     # Embed the files needed by the extended testsuite at runtime. Also include
1183     # the unit tests needed by TEST-02-UNITTESTS. This is mostly equivalent to
1184     # what `ninja install` does for the tests when '-Dinstall-tests=true'.
1185     #
1186     # Why? openSUSE ships a package named 'systemd-testsuite' which contains
1187     # the minimal set of files that allows to run the testsuite on the host (as
1188     # long as it runs an equivalent version of systemd) getting rid of the
1189     # hassles of fetching, configuring, building the source code.
1190     dinfo "Install the files needed by the tests at runtime"
1191     image_install "${SOURCE_DIR}"/test-*
1192     inst_recursive "${SOURCE_DIR}/testdata"
1193     inst_recursive "${SOURCE_DIR}/manual"
1194
1195     # On openSUSE, this directory is not created at package install, at least
1196     # for now.
1197     mkdir -p "$initdir/var/log/journal/remote"
1198 }
1199
1200 install_distro_systemd() {
1201     dinfo "Install distro systemd"
1202
1203     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1204         install_debian_systemd
1205     elif get_bool "$LOOKS_LIKE_SUSE"; then
1206         install_suse_systemd
1207     else
1208         dfatal "NO_BUILD not supported for this distro"
1209         exit 1
1210     fi
1211 }
1212
1213 install_systemd() {
1214     dinfo "Install systemd"
1215     if get_bool "$NO_BUILD"; then
1216         install_distro_systemd
1217     else
1218         install_compiled_systemd
1219     fi
1220
1221     # remove unneeded documentation
1222     rm -fr "${initdir:?}"/usr/share/{man,doc}
1223
1224     # enable debug logging in PID1
1225     echo LogLevel=debug >>"$initdir/etc/systemd/system.conf"
1226     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1227         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf"
1228     fi
1229     # store coredumps in journal
1230     echo Storage=journal >>"$initdir/etc/systemd/coredump.conf"
1231     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1232     mkdir "$initdir/etc/systemd/system/user@.service.d/"
1233     echo -e "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/override.conf"
1234
1235     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1236     # images, since it prevents gcov to write the coverage reports (*.gcda
1237     # files)
1238     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1239         mkdir -p "$initdir/etc/systemd/system/service.d/"
1240         echo -e "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1241         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image
1242         # to make the coverage work with units utilizing DynamicUser=yes. Do
1243         # this only for services from TEST-20, as setting this system-wide
1244         # has many undesirable side-effects
1245         mkdir -p "$initdir/etc/systemd/system/test20-.service.d/"
1246         echo -e "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/test20-.service.d/99-gcov-rwpaths-override.conf"
1247     fi
1248
1249     # If we're built with -Dportabled=false, tests with systemd-analyze
1250     # --profile will fail. Since we need just the profile (text) files, let's
1251     # copy them into the image if they don't exist there.
1252     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1253     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1254         dinfo "Couldn't find portable profiles in the test image"
1255         dinfo "Copying them directly from the source tree"
1256         mkdir -p "$portable_dir"
1257         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1258     fi
1259 }
1260
1261 get_ldpath() {
1262     local rpath
1263     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1264
1265     if [ -z "$rpath" ] ; then
1266         echo "$BUILD_DIR"
1267     else
1268         echo "$rpath"
1269     fi
1270 }
1271
1272 install_missing_libraries() {
1273     dinfo "Install missing libraries"
1274     # install possible missing libraries
1275     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/{,manual/,unsafe/}}*; do
1276         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1277     done
1278
1279     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1280     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1281     # if not present
1282     # See: https://github.com/systemd/systemd/pull/23858
1283     while read -r libgcc_s; do
1284         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1285     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1286
1287     local lib path
1288     # A number of dependencies is now optional via dlopen, so the install
1289     # script will not pick them up, since it looks at linkage.
1290     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1; do
1291         ddebug "Searching for $lib via pkg-config"
1292         if pkg-config --exists "$lib"; then
1293                 path="$(pkg-config --variable=libdir "$lib")"
1294                 if [ -z "${path}" ]; then
1295                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1296                     continue
1297                 fi
1298
1299                 if ! [[ ${lib} =~ ^lib ]]; then
1300                         lib="lib${lib}"
1301                 fi
1302                 # p11-kit-1's .so doesn't have the API level in the name
1303                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1304                         lib="libp11-kit"
1305                 fi
1306                 # Some pkg-config files are broken and give out the wrong paths
1307                 # (eg: libcryptsetup), so just ignore them
1308                 inst_libs "${path}/${lib}.so" || true
1309                 inst_library "${path}/${lib}.so" || true
1310
1311                 if [[ "$lib" == "libxkbcommon" ]]; then
1312                     install_x11_keymaps full
1313                 fi
1314         else
1315             ddebug "$lib.pc not found, skipping"
1316             continue
1317         fi
1318     done
1319
1320     # Install extra openssl 3 stuff
1321     path="$(pkg-config --variable=libdir libcrypto)"
1322     inst_simple "${path}/ossl-modules/legacy.so" || true
1323     inst_simple "${path}/ossl-modules/fips.so" || true
1324     inst_simple "${path}/engines-3/afalg.so" || true
1325     inst_simple "${path}/engines-3/capi.so" || true
1326     inst_simple "${path}/engines-3/loader_attic.so" || true
1327     inst_simple "${path}/engines-3/padlock.so" || true
1328
1329     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1330     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1331     # shellcheck disable=SC2046
1332     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1333 }
1334
1335 cleanup_loopdev() {
1336     if [ -n "${LOOPDEV:=}" ]; then
1337         ddebug "losetup -d $LOOPDEV"
1338         losetup -d "${LOOPDEV}"
1339         unset LOOPDEV
1340     fi
1341 }
1342
1343 add_at_exit_handler cleanup_loopdev
1344
1345 create_empty_image() {
1346     if [ -z "${IMAGE_NAME:=}" ]; then
1347         echo "create_empty_image: \$IMAGE_NAME not set"
1348         exit 1
1349     fi
1350
1351     # Partition sizes are in MiBs
1352     local root_size=1000
1353     local data_size=50
1354     if ! get_bool "$NO_BUILD"; then
1355         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1356             root_size=$((root_size+=200))
1357         fi
1358         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1359             root_size=$((root_size+=200))
1360         fi
1361         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1362             root_size=$((root_size+=250))
1363         fi
1364     fi
1365     if ! get_bool "$STRIP_BINARIES"; then
1366         root_size=$((4 * root_size))
1367         data_size=$((2 * data_size))
1368     fi
1369     if [ "$IMAGE_NAME" = "repart" ]; then
1370         root_size=$((root_size+=1000))
1371     fi
1372
1373     echo "Setting up ${IMAGE_PUBLIC:?} (${root_size} MB)"
1374     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1375
1376     # Create the blank file to use as a root filesystem
1377     truncate -s "${root_size}M" "$IMAGE_PUBLIC"
1378
1379     LOOPDEV=$(losetup --show -P -f "$IMAGE_PUBLIC")
1380     [ -b "$LOOPDEV" ] || return 1
1381     # Create two partitions - a root one and a data one (utilized by some tests)
1382     sfdisk "$LOOPDEV" <<EOF
1383 label: gpt
1384 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=root size=$((root_size - data_size))M bootable
1385 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1386 EOF
1387
1388     udevadm settle
1389
1390     local label=(-L systemd_boot)
1391     # mkfs.reiserfs doesn't know -L. so, use --label instead
1392     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1393     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p1" -q; then
1394         dfatal "Failed to mkfs -t ${FSTYPE}"
1395         exit 1
1396     fi
1397 }
1398
1399 mount_initdir() {
1400     if [ -z "${LOOPDEV:=}" ]; then
1401         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1402         LOOPDEV="$(losetup --show -P -f "$image")"
1403         [ -b "$LOOPDEV" ] || return 1
1404
1405         udevadm settle
1406     fi
1407
1408     if ! mountpoint -q "${initdir:?}"; then
1409         mkdir -p "$initdir"
1410         mount "${LOOPDEV}p1" "$initdir"
1411         TEST_SETUP_CLEANUP_ROOTDIR=1
1412     fi
1413 }
1414
1415 cleanup_initdir() {
1416     # only umount if create_empty_image_rootdir() was called to mount it
1417     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1418         _umount_dir "${initdir:?}"
1419     fi
1420 }
1421
1422 umount_loopback() {
1423     # unmount the loopback device from all places. Otherwise we risk file
1424     # system corruption.
1425     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1426         ddebug "Unmounting all uses of $device"
1427         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1428     done
1429 }
1430
1431 create_empty_image_rootdir() {
1432     create_empty_image
1433     mount_initdir
1434 }
1435
1436 check_asan_reports() {
1437     local ret=0
1438     local root="${1:?}"
1439
1440     if get_bool "$IS_BUILT_WITH_ASAN"; then
1441         ls -l "$root"
1442         if [[ -e "$root/systemd.asan.log.1" ]]; then
1443             cat "$root/systemd.asan.log.1"
1444             ret=$((ret+1))
1445         fi
1446
1447         journald_report="$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)"
1448         if [[ -n "$journald_report" ]]; then
1449             printf "%s\n" "$journald_report"
1450             cat "$root/systemd-journald.out" || :
1451             ret=$((ret+1))
1452         fi
1453
1454         pids="$(
1455             "$JOURNALCTL" -D "$root/var/log/journal" | perl -alne '
1456                  BEGIN {
1457                      %services_to_ignore = (
1458                          "dbus-daemon" => undef,
1459                          "dbus-broker-launch" => undef,
1460                      );
1461                  }
1462                  print $2 if /\s(\S*)\[(\d+)\]:\s*SUMMARY:\s+\w+Sanitizer/ && !exists $services_to_ignore{$1}'
1463         )"
1464         if [[ -n "$pids" ]]; then
1465             ret=$((ret+1))
1466             for pid in $pids; do
1467                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1468             done
1469         fi
1470     fi
1471
1472     return $ret
1473 }
1474
1475 check_coverage_reports() {
1476     local root="${1:?}"
1477
1478     if get_bool "$NO_BUILD"; then
1479         return 0
1480     fi
1481     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1482         return 0
1483     fi
1484
1485     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1486         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1487     else
1488         dest="${TESTDIR:?}/coverage-info"
1489     fi
1490
1491     # Create a coverage report that will later be uploaded. Remove info about
1492     # system libraries/headers, as we don't really care about them.
1493     if [[ -f "$dest" ]]; then
1494         # If the destination report file already exists, don't overwrite it, but
1495         # dump the new report in a temporary file and then merge it with the already
1496         # present one - this usually happens when running both "parts" of a test
1497         # in one run (the qemu and the nspawn part).
1498         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1499         lcov --remove "${dest}.new" -o "${dest}.new" '/usr/include/*' '/usr/lib/*'
1500         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1501         rm -f "${dest}.new"
1502     else
1503         lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}"
1504         lcov --remove "${dest}" -o "${dest}" '/usr/include/*' '/usr/lib/*'
1505     fi
1506
1507     # If the test logs contain lines like:
1508     #
1509     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1510     #
1511     # it means we're possibly missing some coverage since gcov can't write the stats,
1512     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1513     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1514     # `setfacl` stuff in install_compiled_systemd().
1515     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1516        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1517         derror "Detected possibly missing coverage, check the journal"
1518         return 1
1519     fi
1520
1521     return 0
1522 }
1523
1524 save_journal() {
1525     # Default to always saving journal
1526     local save="yes"
1527
1528     if [ "${TEST_SAVE_JOURNAL}" = "no" ]; then
1529         save="no"
1530     elif [ "${TEST_SAVE_JOURNAL}" = "fail" ] && [ "$2" = "0" ]; then
1531         save="no"
1532     fi
1533
1534     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1535         dest="${ARTIFACT_DIRECTORY}/${testname:?}.journal"
1536     else
1537         dest="${TESTDIR:?}/system.journal"
1538     fi
1539
1540     for j in "${1:?}"/*; do
1541         if get_bool "$save"; then
1542             if [ "$SYSTEMD_JOURNAL_REMOTE" = "" ]; then
1543                 cp -a "$j" "$dest"
1544             else
1545                 "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $j"
1546             fi
1547         fi
1548
1549         if [ -n "${TEST_SHOW_JOURNAL}" ]; then
1550             echo "---- $j ----"
1551             "$JOURNALCTL" --no-pager -o short-monotonic --no-hostname --priority="${TEST_SHOW_JOURNAL}" -D "$j"
1552         fi
1553
1554         rm -r "$j"
1555     done
1556
1557     if ! get_bool "$save"; then
1558         return 0
1559     fi
1560
1561     if [ -n "${SUDO_USER}" ]; then
1562         setfacl -m "user:${SUDO_USER:?}:r-X" "$dest"*
1563     fi
1564
1565     # we want to print this sometime later, so save this in a variable
1566     JOURNAL_LIST="$(ls -l "$dest"*)"
1567 }
1568
1569 check_result_common() {
1570     local workspace="${1:?}"
1571     local ret
1572
1573     if [ -s "$workspace/failed" ]; then
1574         # Non-empty …/failed has highest priority
1575         cp -a "$workspace/failed" "${TESTDIR:?}/"
1576         if [ -n "${SUDO_USER}" ]; then
1577             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1578         fi
1579         ret=1
1580     elif get_bool "$TIMED_OUT"; then
1581         echo "(timeout)" >"${TESTDIR:?}/failed"
1582         ret=2
1583     elif [ -e "$workspace/testok" ]; then
1584         # …/testok always counts (but with lower priority than …/failed)
1585         ret=0
1586     elif [ -e "$workspace/skipped" ]; then
1587         # …/skipped always counts (a message is expected)
1588         echo "${TESTNAME:?} was skipped:"
1589         cat "$workspace/skipped"
1590         ret=0
1591     else
1592         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1593         ret=3
1594     fi
1595
1596     check_asan_reports "$workspace" || ret=4
1597
1598     check_coverage_reports "$workspace" || ret=5
1599
1600     save_journal "$workspace/var/log/journal" $ret
1601
1602     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1603         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1604     fi
1605
1606     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1607         echo -n "${TESTNAME:?}: "
1608         cat "$TESTDIR/failed"
1609     fi
1610     echo "${JOURNAL_LIST:-"No journals were saved"}"
1611
1612     return ${ret:?}
1613 }
1614
1615 check_result_nspawn() {
1616     local workspace="${1:?}"
1617     local ret
1618
1619     # Run a test-specific checks if defined by check_result_nspawn_hook()
1620     if declare -F check_result_nspawn_hook >/dev/null; then
1621         if ! check_result_nspawn_hook "${workspace}"; then
1622             derror "check_result_nspawn_hook() returned with EC > 0"
1623             ret=4
1624         fi
1625     fi
1626
1627     check_result_common "${workspace}"
1628     ret=$?
1629
1630     _umount_dir "${initdir:?}"
1631
1632     return $ret
1633 }
1634
1635 # can be overridden in specific test
1636 check_result_qemu() {
1637     local ret
1638     mount_initdir
1639
1640     # Run a test-specific checks if defined by check_result_qemu_hook()
1641     if declare -F check_result_qemu_hook >/dev/null; then
1642         if ! check_result_qemu_hook "${initdir:?}"; then
1643             derror "check_result_qemu_hook() returned with EC > 0"
1644             ret=4
1645         fi
1646     fi
1647
1648     check_result_common "${initdir:?}"
1649     ret=$?
1650
1651     _umount_dir "${initdir:?}"
1652
1653     return $ret
1654 }
1655
1656 check_result_nspawn_unittests() {
1657     local workspace="${1:?}"
1658     local ret=1
1659
1660     [[ -e "$workspace/testok" ]] && ret=0
1661
1662     if [[ -s "$workspace/failed" ]]; then
1663         ret=$((ret + 1))
1664         echo "=== Failed test log ==="
1665         cat "$workspace/failed"
1666     else
1667         if [[ -s "$workspace/skipped" ]]; then
1668             echo "=== Skipped test log =="
1669             cat "$workspace/skipped"
1670             # We might have only skipped tests - that should not fail the job
1671             ret=0
1672         fi
1673         if [[ -s "$workspace/testok" ]]; then
1674             echo "=== Passed tests ==="
1675             cat "$workspace/testok"
1676         fi
1677     fi
1678
1679     get_bool "${TIMED_OUT:=}" && ret=1
1680     check_coverage_reports "$workspace" || ret=5
1681
1682     save_journal "$workspace/var/log/journal" $ret
1683
1684     _umount_dir "${initdir:?}"
1685
1686     return $ret
1687 }
1688
1689 check_result_qemu_unittests() {
1690     local ret=1
1691
1692     mount_initdir
1693     [[ -e "${initdir:?}/testok" ]] && ret=0
1694
1695     if [[ -s "$initdir/failed" ]]; then
1696         ret=$((ret + 1))
1697         echo "=== Failed test log ==="
1698         cat "$initdir/failed"
1699     else
1700         if [[ -s "$initdir/skipped" ]]; then
1701             echo "=== Skipped test log =="
1702             cat "$initdir/skipped"
1703             # We might have only skipped tests - that should not fail the job
1704             ret=0
1705         fi
1706         if [[ -s "$initdir/testok" ]]; then
1707             echo "=== Passed tests ==="
1708             cat "$initdir/testok"
1709         fi
1710     fi
1711
1712     get_bool "${TIMED_OUT:=}" && ret=1
1713     check_coverage_reports "$initdir" || ret=5
1714
1715     save_journal "$initdir/var/log/journal" $ret
1716
1717     _umount_dir "$initdir"
1718
1719     return $ret
1720 }
1721
1722 strip_binaries() {
1723     dinfo "Strip binaries"
1724     if ! get_bool "$STRIP_BINARIES"; then
1725         dinfo "STRIP_BINARIES == no, keeping binaries unstripped"
1726         return 0
1727     fi
1728     while read -r bin; do
1729         strip --strip-unneeded "$bin" |& grep -vi 'file format not recognized' | ddebug || :
1730     done < <(find "${initdir:?}" -executable -not -path '*/lib/modules/*.ko' -type f)
1731 }
1732
1733 create_rc_local() {
1734     dinfo "Create rc.local"
1735     mkdir -p "${initdir:?}/etc/rc.d"
1736     cat >"$initdir/etc/rc.d/rc.local" <<EOF
1737 #!/usr/bin/env bash
1738 exit 0
1739 EOF
1740     chmod 0755 "$initdir/etc/rc.d/rc.local"
1741 }
1742
1743 install_execs() {
1744     ddebug "Install executables from the service files"
1745
1746     local pkg_config_path="${BUILD_DIR:?}/src/core/"
1747     local systemunitdir userunitdir exe
1748     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
1749     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
1750     while read -r exe; do
1751         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
1752         # also, plymouth is pulled in by rescue.service, but even there the exit code
1753         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
1754         dinfo "Attempting to install $exe (based on unit file reference)"
1755         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
1756     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
1757 }
1758
1759 generate_module_dependencies() {
1760     dinfo "Generate modules dependencies"
1761     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
1762         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
1763             dfatal "\"depmod -a $KERNEL_VER\" failed."
1764             exit 1
1765     fi
1766 }
1767
1768 install_depmod_files() {
1769     dinfo "Install depmod files"
1770     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
1771     inst "/lib/modules/$KERNEL_VER/modules.builtin"
1772 }
1773
1774 install_plymouth() {
1775     dinfo "Install plymouth"
1776     # install plymouth, if found... else remove plymouth service files
1777     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
1778     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
1779     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
1780     #         image_install plymouth plymouthd
1781     # else
1782         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
1783     # fi
1784 }
1785
1786 install_haveged() {
1787     # If haveged is installed, it's probably included in initrd and needs to be
1788     # installed in the image too.
1789     if [ -x /usr/sbin/haveged ]; then
1790         dinfo "Install haveged files"
1791         inst /usr/sbin/haveged
1792         for u in /usr/lib/systemd/system/haveged*; do
1793             inst "$u"
1794         done
1795     fi
1796 }
1797
1798 install_ld_so_conf() {
1799     dinfo "Install /etc/ld.so.conf*"
1800     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
1801     ldconfig -r "$initdir"
1802 }
1803
1804 install_testuser() {
1805     dinfo "Set up a test user"
1806     # create unprivileged user for user manager tests
1807     mkdir -p "${initdir:?}/etc/sysusers.d"
1808     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
1809 u testuser    4711     "Test User" /home/testuser
1810 EOF
1811
1812     mkdir -p "$initdir/home/testuser"
1813     chmod 0700 "$initdir/home/testuser"
1814     chown 4711:4711 "$initdir/home/testuser"
1815 }
1816
1817 install_config_files() {
1818     dinfo "Install config files"
1819     inst /etc/sysconfig/init || :
1820     inst /etc/passwd
1821     inst /etc/shadow
1822     inst_any /etc/login.defs /usr/etc/login.defs
1823     inst /etc/group
1824     inst /etc/shells
1825     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
1826     inst /etc/pam.conf || :
1827     inst_any /etc/os-release /usr/lib/os-release
1828     inst /etc/localtime
1829     # we want an empty environment
1830     : >"${initdir:?}/etc/environment"
1831     : >"$initdir/etc/machine-id"
1832     : >"$initdir/etc/resolv.conf"
1833
1834     # set the hostname
1835     echo 'H' >"$initdir/etc/hostname"
1836
1837     # let's set up just one image with the traditional verbose output
1838     if [ "${IMAGE_NAME:?}" != "basic" ]; then
1839         mkdir -p "$initdir/etc/systemd/system.conf.d"
1840         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
1841     fi
1842 }
1843
1844 install_basic_tools() {
1845     dinfo "Install basic tools"
1846     image_install "${BASICTOOLS[@]}"
1847     image_install -o sushell
1848     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
1849     image_install -o ldconfig.real
1850 }
1851
1852 install_debug_tools() {
1853     dinfo "Install debug tools"
1854     image_install -o "${DEBUGTOOLS[@]}"
1855
1856     if get_bool "$INTERACTIVE_DEBUG"; then
1857         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
1858         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
1859         mkdir -p "$getty_override"
1860         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
1861         echo 'export TERM=linux' >>"$initdir/etc/profile"
1862
1863         if command -v resize >/dev/null; then
1864             image_install resize
1865             echo "resize" >>"$initdir/etc/profile"
1866         fi
1867
1868         # Sometimes we might end up with plymouthd still running (especially
1869         # with the initrd -> asan_wrapper -> systemd transition), which will eat
1870         # our inputs and make debugging via tty impossible. Let's fix this by
1871         # killing plymouthd explicitly for the interactive sessions.
1872         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
1873         #   - https://github.com/llvm/llvm-project/issues/49223
1874         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
1875         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
1876         cat >"$plymouth_unit" <<EOF
1877 [Unit]
1878 After=multi-user.target
1879
1880 [Service]
1881 ExecStart=sh -c 'killall --verbose plymouthd || :'
1882
1883 [Install]
1884 WantedBy=multi-user.target
1885 EOF
1886         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
1887     fi
1888 }
1889
1890 install_libnss() {
1891     dinfo "Install libnss"
1892     # install libnss_files for login
1893     local NSS_LIBS
1894     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
1895     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
1896         image_install "${NSS_LIBS[@]}"
1897     fi
1898 }
1899
1900 install_dbus() {
1901     dinfo "Install dbus"
1902     inst "${ROOTLIBDIR:?}/system/dbus.socket"
1903
1904     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1905     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
1906         inst "$ROOTLIBDIR/system/dbus-broker.service"
1907         inst_symlink /etc/systemd/system/dbus.service
1908         inst /usr/bin/dbus-broker
1909         inst /usr/bin/dbus-broker-launch
1910     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
1911         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1912         inst "$ROOTLIBDIR/system/dbus-daemon.service"
1913         # Alias symlink
1914         inst_symlink /etc/systemd/system/dbus.service
1915     else
1916         inst "$ROOTLIBDIR/system/dbus.service"
1917     fi
1918
1919     while read -r file; do
1920         inst "$file"
1921     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
1922
1923     # setup policy for Type=dbus test
1924     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
1925     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
1926 <?xml version="1.0"?>
1927 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
1928         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
1929 <busconfig>
1930     <policy user="root">
1931         <allow own="systemd.test.ExecStopPost"/>
1932     </policy>
1933 </busconfig>
1934 EOF
1935
1936     # If we run without KVM, bump the service start timeout
1937     if ! get_bool "$QEMU_KVM"; then
1938         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
1939 <?xml version="1.0"?>
1940 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
1941         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
1942 <busconfig>
1943     <limit name="service_start_timeout">60000</limit>
1944 </busconfig>
1945 EOF
1946         # Bump the client-side timeout in sd-bus as well
1947         mkdir -p "$initdir/etc/systemd/system.conf.d"
1948         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=60' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
1949     fi
1950 }
1951
1952 install_user_dbus() {
1953     dinfo "Install user dbus"
1954     local userunitdir
1955     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
1956         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
1957         userunitdir=/usr/lib/systemd/user
1958     fi
1959
1960     inst "$userunitdir/dbus.socket"
1961     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
1962
1963     # Append the After= dependency on dbus in case it isn't already set up
1964     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
1965     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
1966 [Unit]
1967 After=dbus.service
1968 EOF
1969
1970     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1971     if [ -f "$userunitdir/dbus-broker.service" ]; then
1972         inst "$userunitdir/dbus-broker.service"
1973         inst_symlink /etc/systemd/user/dbus.service
1974     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
1975         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1976         inst "$userunitdir/dbus-daemon.service"
1977         # Alias symlink
1978         inst_symlink /etc/systemd/user/dbus.service
1979     else
1980         inst "$userunitdir/dbus.service"
1981     fi
1982 }
1983
1984 install_pam() {
1985     dinfo "Install PAM"
1986     local paths=()
1987
1988     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
1989         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
1990     else
1991         paths+=(/lib*/security)
1992     fi
1993
1994     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
1995         [ -d "$d" ] && paths+=("$d")
1996     done
1997
1998     while read -r file; do
1999         inst "$file"
2000     done < <(find "${paths[@]}" -xtype f)
2001
2002     # pam_unix depends on unix_chkpwd.
2003     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2004     image_install -o unix_chkpwd
2005
2006     # set empty root password for easy debugging
2007     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2008
2009     # And make sure pam_unix will accept it by making sure that
2010     # the PAM module has the nullok option.
2011     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2012         [ -d "$initdir/$d" ] || continue
2013         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2014     done
2015 }
2016
2017 install_locales() {
2018     # install only C.UTF-8 and English locales
2019     dinfo "Install locales"
2020
2021     if command -v meson >/dev/null \
2022             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2023             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2024         # locale-gen support
2025         image_install -o locale-gen localedef
2026         inst /etc/locale.gen || :
2027         inst /usr/share/i18n/SUPPORTED || :
2028         inst_recursive /usr/share/i18n/charmaps
2029         inst_recursive /usr/share/i18n/locales
2030         inst_recursive /usr/share/locale/en*
2031         inst_recursive /usr/share/locale/de*
2032         image_install /usr/share/locale/locale.alias
2033         # locale-gen might either generate each locale separately or merge them
2034         # into a single archive
2035         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2036               image_install /usr/lib/locale/locale-archive); then
2037             dfatal "Failed to install required locales"
2038             exit 1
2039         fi
2040     else
2041         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2042     fi
2043 }
2044
2045 # shellcheck disable=SC2120
2046 install_keymaps() {
2047     local i p
2048     local -a prefix=(
2049         "/usr/lib"
2050         "/usr/share"
2051     )
2052
2053     dinfo "Install console keymaps"
2054
2055     if command -v meson >/dev/null \
2056             && [[ "$(meson configure "${BUILD_DIR:?}" | grep 'split-usr' | awk '{ print $2 }')" == "true" ]] \
2057             || [[ ! -L /lib ]]; then
2058         prefix+=(
2059             "/lib"
2060         )
2061     fi
2062
2063     if (( $# == 0 )); then
2064         for p in "${prefix[@]}"; do
2065             # The first three paths may be deprecated.
2066             # It seems now the last three paths are used by many distributions.
2067             for i in \
2068                 "$p"/kbd/keymaps/include/* \
2069                 "$p"/kbd/keymaps/i386/include/* \
2070                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2071                 "$p"/kbd/keymaps/legacy/include/* \
2072                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2073                 "$p"/kbd/keymaps/xkb/us*; do
2074                     [[ -f "$i" ]] || continue
2075                     inst "$i"
2076             done
2077         done
2078     else
2079         # When it takes any argument, then install more keymaps.
2080         for p in "${prefix[@]}"; do
2081             for i in \
2082                 "$p"/kbd/keymaps/include/* \
2083                 "$p"/kbd/keymaps/i386/*/* \
2084                 "$p"/kbd/keymaps/legacy/i386/*/* \
2085                 "$p"/kbd/keymaps/xkb/*; do
2086                     [[ -f "$i" ]] || continue
2087                     inst "$i"
2088             done
2089         done
2090     fi
2091 }
2092
2093 install_x11_keymaps() {
2094     dinfo "Install x11 keymaps"
2095
2096     if (( $# == 0 )); then
2097         # Install only keymap list.
2098         inst /usr/share/X11/xkb/rules/base.lst
2099     else
2100         # When it takes any argument, then install all keymaps.
2101         inst_recursive /usr/share/X11/xkb
2102     fi
2103 }
2104
2105 install_zoneinfo() {
2106     dinfo "Install time zones"
2107     inst_any /usr/share/zoneinfo/Asia/Seoul
2108     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2109     inst_any /usr/share/zoneinfo/Australia/Sydney
2110     inst_any /usr/share/zoneinfo/Europe/Berlin
2111     inst_any /usr/share/zoneinfo/Europe/Dublin
2112     inst_any /usr/share/zoneinfo/Europe/Kiev
2113     inst_any /usr/share/zoneinfo/Pacific/Auckland
2114     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2115     inst_any /usr/share/zoneinfo/CET
2116     inst_any /usr/share/zoneinfo/EET
2117     inst_any /usr/share/zoneinfo/UTC
2118 }
2119
2120 install_fonts() {
2121     dinfo "Install system fonts"
2122     for i in \
2123         /usr/lib/kbd/consolefonts/eurlatgr* \
2124         /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
2125             [[ -f "$i" ]] || continue
2126             inst "$i"
2127     done
2128 }
2129
2130 install_terminfo() {
2131     dinfo "Install terminfo files"
2132     local terminfodir
2133     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2134         [ -f "${terminfodir}/l/linux" ] && break
2135     done
2136     image_install -o "${terminfodir}/l/linux"
2137 }
2138
2139 has_user_dbus_socket() {
2140     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2141         return 0
2142     else
2143         echo "Per-user instances are not supported. Skipping..."
2144         return 1
2145     fi
2146 }
2147
2148 setup_nspawn_root_hook() { :;}
2149
2150 setup_nspawn_root() {
2151     if [ -z "${initdir}" ]; then
2152         dfatal "\$initdir not defined"
2153         exit 1
2154     fi
2155
2156     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2157
2158     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2159         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2160         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2161     fi
2162
2163     setup_nspawn_root_hook
2164 }
2165
2166 setup_basic_dirs() {
2167     mkdir -p "${initdir:?}/run"
2168     mkdir -p "$initdir/etc/systemd/system"
2169     mkdir -p "$initdir/var/log/journal"
2170
2171
2172     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2173         if [ -L "/$d" ]; then
2174             inst_symlink "/$d"
2175         else
2176             inst_dir "/$d"
2177         fi
2178     done
2179
2180     ln -sfn /run "$initdir/var/run"
2181     ln -sfn /run/lock "$initdir/var/lock"
2182 }
2183
2184 mask_supporting_services() {
2185     # mask some services that we do not want to run in these tests
2186     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2187     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2188     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2189     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2190     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2191 }
2192
2193 inst_libs() {
2194     local bin="${1:?}"
2195     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2196     local file line
2197
2198     while read -r line; do
2199         [[ "$line" = 'not a dynamic executable' ]] && break
2200         # Ignore errors about our own stuff missing. This is most likely caused
2201         # by ldd attempting to use the unprefixed RPATH.
2202         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2203
2204         if [[ "$line" =~ $so_regex ]]; then
2205             file="${BASH_REMATCH[1]}"
2206             [[ -e "${initdir:?}/$file" ]] && continue
2207             inst_library "$file"
2208             continue
2209         fi
2210
2211         if [[ "$line" =~ not\ found ]]; then
2212             dfatal "Missing a shared library required by $bin."
2213             dfatal "Run \"ldd $bin\" to find out what it is."
2214             dfatal "$line"
2215             dfatal "Cannot create a test image."
2216             exit 1
2217         fi
2218     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2219 }
2220
2221 import_testdir() {
2222     # make sure we don't get a stale LOOPDEV value from old times
2223     local _LOOPDEV="${LOOPDEV:=}"
2224     # We don't want shellcheck to follow & check the $STATEFILE
2225     # shellcheck source=/dev/null
2226     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2227     LOOPDEV="$_LOOPDEV"
2228     if [[ ! -d "$TESTDIR" ]]; then
2229         if [[ -z "$TESTDIR" ]]; then
2230             TESTDIR="$(mktemp --tmpdir=/var/tmp -d -t systemd-test.XXXXXX)"
2231         else
2232             mkdir -p "$TESTDIR"
2233         fi
2234
2235         cat >"$STATEFILE" <<EOF
2236 TESTDIR="$TESTDIR"
2237 EOF
2238         export TESTDIR
2239     fi
2240
2241     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2242     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2243 }
2244
2245 import_initdir() {
2246     initdir="${TESTDIR:?}/root"
2247     mkdir -p "$initdir"
2248     export initdir
2249 }
2250
2251 get_cgroup_hierarchy() {
2252     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2253         cgroup2fs)
2254             echo "unified"
2255             ;;
2256         tmpfs)
2257             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2258                 echo "hybrid"
2259             else
2260                 echo "legacy"
2261             fi
2262             ;;
2263         *)
2264             dfatal "Failed to determine host's cgroup hierarchy"
2265             exit 1
2266     esac
2267 }
2268
2269 ## @brief Converts numeric logging level to the first letter of level name.
2270 #
2271 # @param lvl Numeric logging level in range from 1 to 6.
2272 # @retval 1 if @a lvl is out of range.
2273 # @retval 0 if @a lvl is correct.
2274 # @result Echoes first letter of level name.
2275 _lvl2char() {
2276     case "$1" in
2277         1) echo F;;
2278         2) echo E;;
2279         3) echo W;;
2280         4) echo I;;
2281         5) echo D;;
2282         6) echo T;;
2283         *) return 1;;
2284     esac
2285 }
2286
2287 ## @brief Internal helper function for _do_dlog()
2288 #
2289 # @param lvl Numeric logging level.
2290 # @param msg Message.
2291 # @retval 0 It's always returned, even if logging failed.
2292 #
2293 # @note This function is not supposed to be called manually. Please use
2294 # dtrace(), ddebug(), or others instead which wrap this one.
2295 #
2296 # This function calls _do_dlog() either with parameter msg, or if
2297 # none is given, it will read standard input and will use every line as
2298 # a message.
2299 #
2300 # This enables:
2301 # dwarn "This is a warning"
2302 # echo "This is a warning" | dwarn
2303 LOG_LEVEL="${LOG_LEVEL:-4}"
2304
2305 dlog() {
2306     local lvl lvlc
2307
2308     [ -z "$LOG_LEVEL" ] && return 0
2309     lvl="${1:?}"; shift
2310     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2311     lvlc="$(_lvl2char "$lvl")" || return 0
2312
2313     if [ $# -ge 1 ]; then
2314         echo "$lvlc: $*"
2315     else
2316         while read -r line; do
2317             echo "$lvlc: " "$line"
2318         done
2319     fi
2320 }
2321
2322 ## @brief Logs message at TRACE level (6)
2323 #
2324 # @param msg Message.
2325 # @retval 0 It's always returned, even if logging failed.
2326 dtrace() {
2327     set +x
2328     dlog 6 "$@"
2329     if get_bool "${debug:=}"; then
2330         set -x
2331     fi
2332 }
2333
2334 ## @brief Logs message at DEBUG level (5)
2335 #
2336 # @param msg Message.
2337 # @retval 0 It's always returned, even if logging failed.
2338 ddebug() {
2339     dlog 5 "$@"
2340 }
2341
2342 ## @brief Logs message at INFO level (4)
2343 #
2344 # @param msg Message.
2345 # @retval 0 It's always returned, even if logging failed.
2346 dinfo() {
2347     set +x
2348     dlog 4 "$@"
2349     if get_bool "${debug:=}"; then
2350         set -x
2351     fi
2352 }
2353
2354 ## @brief Logs message at WARN level (3)
2355 #
2356 # @param msg Message.
2357 # @retval 0 It's always returned, even if logging failed.
2358 dwarn() {
2359     set +x
2360     dlog 3 "$@"
2361     if get_bool "${debug:=}"; then
2362         set -x
2363     fi
2364 }
2365
2366 ## @brief Logs message at ERROR level (2)
2367 #
2368 # @param msg Message.
2369 # @retval 0 It's always returned, even if logging failed.
2370 derror() {
2371     dlog 2 "$@"
2372 }
2373
2374 ## @brief Logs message at FATAL level (1)
2375 #
2376 # @param msg Message.
2377 # @retval 0 It's always returned, even if logging failed.
2378 dfatal() {
2379     set +x
2380     dlog 1 "$@"
2381     if get_bool "${debug:=}"; then
2382         set -x
2383     fi
2384 }
2385
2386
2387 # Generic substring function.  If $2 is in $1, return 0.
2388 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2389
2390 # normalize_path <path>
2391 # Prints the normalized path, where it removes any duplicated
2392 # and trailing slashes.
2393 # Example:
2394 # $ normalize_path ///test/test//
2395 # /test/test
2396 normalize_path() {
2397     shopt -q -s extglob
2398     set -- "${1//+(\/)//}"
2399     shopt -q -u extglob
2400     echo "${1%/}"
2401 }
2402
2403 # convert_abs_rel <from> <to>
2404 # Prints the relative path, when creating a symlink to <to> from <from>.
2405 # Example:
2406 # $ convert_abs_rel /usr/bin/test /bin/test-2
2407 # ../../bin/test-2
2408 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2409 convert_abs_rel() {
2410     local __current __absolute __abssize __cursize __newpath
2411     local -i __i __level
2412
2413     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2414
2415     # corner case #1 - self looping link
2416     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2417
2418     # corner case #2 - own dir link
2419     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2420
2421     IFS="/" read -ra __current <<< "$1"
2422     IFS="/" read -ra __absolute <<< "$2"
2423
2424     __abssize=${#__absolute[@]}
2425     __cursize=${#__current[@]}
2426
2427     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2428         (( __level++ ))
2429         if (( __level > __abssize || __level > __cursize ))
2430         then
2431             break
2432         fi
2433     done
2434
2435     for ((__i = __level; __i < __cursize-1; __i++)); do
2436         if ((__i > __level))
2437         then
2438             __newpath=$__newpath"/"
2439         fi
2440         __newpath=$__newpath".."
2441     done
2442
2443     for ((__i = __level; __i < __abssize; __i++)); do
2444         if [[ -n $__newpath ]]
2445         then
2446             __newpath=$__newpath"/"
2447         fi
2448         __newpath=$__newpath${__absolute[__i]}
2449     done
2450
2451     echo "$__newpath"
2452 }
2453
2454
2455 # Install a directory, keeping symlinks as on the original system.
2456 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2457 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2458 # and a symlink ${initdir}/lib -> lib64.
2459 inst_dir() {
2460     local dir="${1:?}"
2461     local part="${dir%/*}"
2462     local file
2463
2464     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2465
2466     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2467         dir="$part $dir"
2468         part="${part%/*}"
2469     done
2470
2471     # iterate over parent directories
2472     for file in $dir; do
2473         [[ -e "${initdir}/$file" ]] && continue
2474         if [[ -L $file ]]; then
2475             inst_symlink "$file"
2476         else
2477             # create directory
2478             mkdir -m 0755 "${initdir}/$file" || return 1
2479             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2480             chmod u+w "${initdir}/$file"
2481         fi
2482     done
2483 }
2484
2485 # $1 = file to copy to ramdisk
2486 # $2 (optional) Name for the file on the ramdisk
2487 # Location of the image dir is assumed to be $initdir
2488 # We never overwrite the target if it exists.
2489 inst_simple() {
2490     [[ -f "${1:?}" ]] || return 1
2491     strstr "$1" "/" || return 1
2492
2493     local src="$1"
2494     local target="${2:-$1}"
2495     if ! [[ -d ${initdir:?}/$target ]]; then
2496         [[ -e ${initdir}/$target ]] && return 0
2497         [[ -L ${initdir}/$target ]] && return 0
2498         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2499     fi
2500     # install checksum files also
2501     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2502         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2503     fi
2504     ddebug "Installing $src"
2505     cp --sparse=always -pfL "$src" "${initdir}/$target"
2506 }
2507
2508 # find symlinks linked to given library file
2509 # $1 = library file
2510 # Function searches for symlinks by stripping version numbers appended to
2511 # library filename, checks if it points to the same target and finally
2512 # prints the list of symlinks to stdout.
2513 #
2514 # Example:
2515 # rev_lib_symlinks libfoo.so.8.1
2516 # output: libfoo.so.8 libfoo.so
2517 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2518 rev_lib_symlinks() {
2519     local fn="${1:?}"
2520     local links=""
2521     local orig
2522     orig="$(readlink -f "$1")"
2523
2524     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2525
2526     until [[ "${fn##*.}" == so ]]; do
2527         fn="${fn%.*}"
2528         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2529     done
2530
2531     echo "${links}"
2532 }
2533
2534 # Same as above, but specialized to handle dynamic libraries.
2535 # It handles making symlinks according to how the original library
2536 # is referenced.
2537 inst_library() {
2538     local src="${1:?}"
2539     local dest="${2:-$1}"
2540     local reallib symlink
2541
2542     strstr "$1" "/" || return 1
2543     [[ -e ${initdir:?}/$dest ]] && return 0
2544     if [[ -L $src ]]; then
2545         # install checksum files also
2546         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2547             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2548         fi
2549         reallib="$(readlink -f "$src")"
2550         inst_simple "$reallib" "$reallib"
2551         inst_dir "${dest%/*}"
2552         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2553         ddebug "Creating symlink $reallib -> $dest"
2554         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2555     else
2556         inst_simple "$src" "$dest"
2557     fi
2558
2559     # Create additional symlinks.  See rev_symlinks description.
2560     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2561         if [[ ! -e "$initdir/$symlink" ]]; then
2562             ddebug "Creating extra symlink: $symlink"
2563             inst_symlink "$symlink"
2564         fi
2565     done
2566 }
2567
2568 # find a binary.  If we were not passed the full path directly,
2569 # search in the usual places to find the binary.
2570 find_binary() {
2571     local bin="${1:?}"
2572     if [[ -z ${bin##/*} ]]; then
2573         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2574             echo "$bin"
2575             return 0
2576         fi
2577     fi
2578
2579     type -P "$bin"
2580 }
2581
2582 # Same as above, but specialized to install binary executables.
2583 # Install binary executable, and all shared library dependencies, if any.
2584 inst_binary() {
2585     local bin="${1:?}"
2586     local path target
2587
2588     # In certain cases we might attempt to install a binary which is already
2589     # present in the test image, yet it's missing from the host system.
2590     # In such cases, let's check if the binary indeed exists in the image
2591     # before doing any other checks. If it does, immediately return with
2592     # success.
2593     if [[ $# -eq 1 ]]; then
2594         for path in "" bin sbin usr/bin usr/sbin; do
2595             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2596         done
2597     fi
2598
2599     bin="$(find_binary "$bin")" || return 1
2600     target="${2:-$bin}"
2601     [[ -e "${initdir:?}/$target" ]] && return 0
2602     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2603
2604     local file line
2605     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2606     # DSOs provided by systemd
2607     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2608     local wrap_binary=0
2609     # I love bash!
2610     while read -r line; do
2611         [[ "$line" = 'not a dynamic executable' ]] && break
2612
2613         # Ignore errors about our own stuff missing. This is most likely caused
2614         # by ldd attempting to use the unprefixed RPATH.
2615         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2616
2617         # We're built with ASan and the target binary loads one of the systemd's
2618         # DSOs, so we need to tweak the environment before executing the binary
2619         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2620             wrap_binary=1
2621         fi
2622
2623         if [[ "$line" =~ $so_regex ]]; then
2624             file="${BASH_REMATCH[1]}"
2625             [[ -e "${initdir}/$file" ]] && continue
2626             inst_library "$file"
2627             continue
2628         fi
2629
2630         if [[ "$line" =~ not\ found ]]; then
2631             dfatal "Missing a shared library required by $bin."
2632             dfatal "Run \"ldd $bin\" to find out what it is."
2633             dfatal "$line"
2634             dfatal "Cannot create a test image."
2635             exit 1
2636         fi
2637     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2638
2639     # Same as above, but we need to wrap certain libraries unconditionally
2640     #
2641     # chown, getent, login, su, useradd, userdel - dlopen()s (not only) systemd's PAM modules
2642     # ls, stat - pulls in nss_systemd with certain options (like ls -l) when
2643     #            nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2644     # delv, dig - pulls in nss_resolve if `resolve` is in nsswitch.conf
2645     # tar - called by machinectl in TEST-25
2646     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ /(chown|delv|dig|getent|login|ls|stat|su|tar|useradd|userdel)$ ]]; then
2647         wrap_binary=1
2648     fi
2649
2650     # If the target binary is built with ASan support, we don't need to wrap
2651     # it, as it should handle everything by itself
2652     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2653         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2654         # Install the target binary with a ".orig" suffix
2655         inst_simple "$bin" "${target}.orig"
2656         # Create a simple shell wrapper in place of the target binary, which
2657         # sets necessary ASan-related env variables and then exec()s the
2658         # suffixed target binary
2659         cat >"$initdir/$target" <<EOF
2660 #!/bin/bash
2661 # Preload the ASan runtime DSO, otherwise ASAn will complain
2662 export LD_PRELOAD="$ASAN_RT_PATH"
2663 # Disable LSan to speed things up, since we don't care about leak reports
2664 # from 'external' binaries
2665 export ASAN_OPTIONS=detect_leaks=0
2666 # Set argv[0] to the original binary name without the ".orig" suffix
2667 exec -a "\$0" -- "${target}.orig" "\$@"
2668 EOF
2669         chmod +x "$initdir/$target"
2670     else
2671         inst_simple "$bin" "$target"
2672     fi
2673 }
2674
2675 # same as above, except for shell scripts.
2676 # If your shell script does not start with shebang, it is not a shell script.
2677 inst_script() {
2678     local bin line shebang_regex
2679     bin="$(find_binary "${1:?}")" || return 1
2680     shift
2681
2682     read -r -n 80 line <"$bin"
2683     # If debug is set, clean unprintable chars to prevent messing up the term
2684     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2685     shebang_regex='(#! *)(/[^ ]+).*'
2686     [[ "$line" =~ $shebang_regex ]] || return 1
2687     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2688 }
2689
2690 # same as above, but specialized for symlinks
2691 inst_symlink() {
2692     local src="${1:?}"
2693     local target="${2:-$src}"
2694     local realsrc
2695
2696     strstr "$src" "/" || return 1
2697     [[ -L "$src" ]] || return 1
2698     [[ -L "${initdir:?}/$target" ]] && return 0
2699     realsrc="$(readlink -f "$src")"
2700     if ! [[ -e "$initdir/$realsrc" ]]; then
2701         if [[ -d "$realsrc" ]]; then
2702             inst_dir "$realsrc"
2703         else
2704             inst "$realsrc"
2705         fi
2706     fi
2707     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
2708     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
2709     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
2710 }
2711
2712 # attempt to install any programs specified in a udev rule
2713 inst_rule_programs() {
2714     local rule="${1:?}"
2715     local prog bin
2716
2717     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
2718         if [ -x "/lib/udev/$prog" ]; then
2719             bin="/lib/udev/$prog"
2720         else
2721             if ! bin="$(find_binary "$prog")"; then
2722                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
2723                 continue
2724             fi
2725         fi
2726
2727         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
2728         image_install "$bin"
2729     done
2730 }
2731
2732 # udev rules always get installed in the same place, so
2733 # create a function to install them to make life simpler.
2734 inst_rules() {
2735     local target=/etc/udev/rules.d
2736     local found rule
2737
2738     inst_dir "/lib/udev/rules.d"
2739     inst_dir "$target"
2740     for rule in "$@"; do
2741         if [ "${rule#/}" = "$rule" ]; then
2742             for r in /lib/udev/rules.d /etc/udev/rules.d; do
2743                 if [[ -f "$r/$rule" ]]; then
2744                     found="$r/$rule"
2745                     inst_simple "$found"
2746                     inst_rule_programs "$found"
2747                 fi
2748             done
2749         fi
2750         for r in '' ./; do
2751             if [[ -f "${r}${rule}" ]]; then
2752                 found="${r}${rule}"
2753                 inst_simple "$found" "$target/${found##*/}"
2754                 inst_rule_programs "$found"
2755             fi
2756         done
2757         [[ $found ]] || dinfo "Skipping udev rule: $rule"
2758         found=
2759     done
2760 }
2761
2762 # general purpose installation function
2763 # Same args as above.
2764 inst() {
2765     case $# in
2766         1) ;;
2767         2)
2768             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
2769             [[ "$initdir" = "$2" ]] && set "$1"
2770             ;;
2771         3)
2772             [[ -z "$initdir" ]] && export initdir="$2"
2773             set "$1" "$3"
2774             ;;
2775         *)
2776             dfatal "inst only takes 1 or 2 or 3 arguments"
2777             exit 1
2778             ;;
2779     esac
2780
2781     local fun
2782     for fun in inst_symlink inst_script inst_binary inst_simple; do
2783         "$fun" "$@" && return 0
2784     done
2785
2786     dwarn "Failed to install '$1'"
2787     return 1
2788 }
2789
2790 # install any of listed files
2791 #
2792 # If first argument is '-d' and second some destination path, first accessible
2793 # source is installed into this path, otherwise it will installed in the same
2794 # path as source.  If none of listed files was installed, function return 1.
2795 # On first successful installation it returns with 0 status.
2796 #
2797 # Example:
2798 #
2799 # inst_any -d /bin/foo /bin/bar /bin/baz
2800 #
2801 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
2802 # initrd.
2803 inst_any() {
2804     local dest file
2805
2806     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
2807
2808     for file in "$@"; do
2809         if [[ -e "$file" ]]; then
2810             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
2811             inst "$file" && return 0
2812         fi
2813     done
2814
2815     return 1
2816 }
2817
2818 inst_recursive() {
2819     local p item
2820
2821     for p in "$@"; do
2822         # Make sure the source exists, as the process substitution below
2823         # suppresses errors
2824         stat "$p" >/dev/null || return 1
2825
2826         while read -r item; do
2827             if [[ -d "$item" ]]; then
2828                 inst_dir "$item"
2829             elif [[ -f "$item" ]]; then
2830                 inst_simple "$item"
2831             fi
2832         done < <(find "$p" 2>/dev/null)
2833     done
2834 }
2835
2836 # image_install [-o ] <file> [<file> ... ]
2837 # Install <file> to the test image
2838 # -o optionally install the <file> and don't fail, if it is not there
2839 image_install() {
2840     local optional=no
2841     local prog="${1:?}"
2842
2843     if [[ "$prog" = '-o' ]]; then
2844         optional=yes
2845         shift
2846     fi
2847
2848     for prog in "$@"; do
2849         if ! inst "$prog" ; then
2850             if get_bool "$optional"; then
2851                 dinfo "Skipping program $prog as it cannot be found and is" \
2852                     "flagged to be optional"
2853             else
2854                 dfatal "Failed to install $prog"
2855                 exit 1
2856             fi
2857         fi
2858     done
2859 }
2860
2861 # Install a single kernel module along with any firmware it may require.
2862 # $1 = full path to kernel module to install
2863 install_kmod_with_fw() {
2864     local module="${1:?}"
2865     # no need to go further if the module is already installed
2866     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
2867     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
2868
2869     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
2870
2871     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
2872
2873     local modname="${module##*/}"
2874     local fwdir found fw
2875     modname="${modname%.ko*}"
2876
2877     while read -r fw; do
2878         found=
2879         for fwdir in /lib/firmware/updates /lib/firmware; do
2880             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
2881                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
2882                 found=yes
2883             fi
2884         done
2885         if ! get_bool "$found"; then
2886             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
2887                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
2888                     "\"${modname}.ko\""
2889             else
2890                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
2891                     "\"${modname}.ko\""
2892             fi
2893         fi
2894     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
2895     return 0
2896 }
2897
2898 # Do something with all the dependencies of a kernel module.
2899 # Note that kernel modules depend on themselves using the technique we use
2900 # $1 = function to call for each dependency we find
2901 #      It will be passed the full path to the found kernel module
2902 # $2 = module to get dependencies for
2903 # rest of args = arguments to modprobe
2904 for_each_kmod_dep() {
2905     local func="${1:?}"
2906     local kmod="${2:?}"
2907     local found=0
2908     local cmd modpath
2909     shift 2
2910
2911     while read -r cmd modpath _; do
2912         [[ "$cmd" = insmod ]] || continue
2913         "$func" "$modpath" || return $?
2914         found=1
2915     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
2916
2917     ! get_bool "$found" && return 1
2918     return 0
2919 }
2920
2921 # instmods [-c] <kernel module> [<kernel module> ... ]
2922 # instmods [-c] <kernel subsystem>
2923 # install kernel modules along with all their dependencies.
2924 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
2925 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
2926 instmods() {
2927     local check=no
2928     if [[ $# -ge 0 && "$1" = '-c' ]]; then
2929         check=yes
2930         shift
2931     fi
2932
2933     inst1mod() {
2934         local mod="${1:?}"
2935         local ret=0
2936         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
2937
2938         case "$mod" in
2939             =*)
2940                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
2941                     (
2942                         [[ "$mpargs" ]] && echo "$mpargs"
2943                         cat "${mod_dir}/modules.${mod#=}"
2944                     ) | instmods
2945                 else
2946                     (
2947                         [[ "$mpargs" ]] && echo "$mpargs"
2948                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
2949                     ) | instmods
2950                 fi
2951                 ;;
2952             --*)
2953                 mpargs+=" $mod"
2954                 ;;
2955             i2o_scsi)
2956                 # Do not load this diagnostic-only module
2957                 return
2958                 ;;
2959             *)
2960                 mod=${mod##*/}
2961                 # if we are already installed, skip this module and go on
2962                 # to the next one.
2963                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
2964
2965                 # We use '-d' option in modprobe only if modules prefix path
2966                 # differs from default '/'.  This allows us to use Dracut with
2967                 # old version of modprobe which doesn't have '-d' option.
2968                 local mod_dirname=${mod_dir%%/lib/modules/*}
2969                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
2970
2971                 # ok, load the module, all its dependencies, and any firmware
2972                 # it may require
2973                 for_each_kmod_dep install_kmod_with_fw "$mod" \
2974                     --set-version "$KERNEL_VER" \
2975                     ${mod_dirname:+"$mod_dirname"} \
2976                     ${mpargs:+"$mpargs"}
2977                 ((ret+=$?))
2978                 ;;
2979         esac
2980         return "$ret"
2981     }
2982
2983     local mod mpargs
2984
2985     if [[ $# -eq 0 ]]; then  # filenames from stdin
2986         while read -r mod; do
2987             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2988                 dfatal "Failed to install $mod"
2989                 return 1
2990             fi
2991         done
2992     fi
2993
2994     for mod in "$@"; do # filenames as arguments
2995         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
2996             dfatal "Failed to install $mod"
2997             return 1
2998         fi
2999     done
3000
3001     return 0
3002 }
3003
3004 _umount_dir() {
3005     local mountpoint="${1:?}"
3006     if mountpoint -q "$mountpoint"; then
3007         ddebug "umount $mountpoint"
3008         umount "$mountpoint"
3009     fi
3010 }
3011
3012 # can be overridden in specific test
3013 test_setup_cleanup() {
3014     cleanup_initdir
3015 }
3016
3017 _test_cleanup() {
3018     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3019     (
3020         set +e
3021         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3022         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3023         # If multiple setups/cleans are ran in parallel, this can cause a race
3024         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3025             rm -vf "${IMAGESTATEDIR}/default.img"
3026         fi
3027         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3028         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3029     ) || :
3030 }
3031
3032 # can be overridden in specific test
3033 test_cleanup() {
3034     _test_cleanup
3035 }
3036
3037 test_cleanup_again() {
3038     [ -n "$TESTDIR" ] || return
3039     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3040     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3041 }
3042
3043 test_create_image() {
3044     create_empty_image_rootdir
3045
3046     # Create what will eventually be our root filesystem onto an overlay
3047     (
3048         LOG_LEVEL=5
3049         setup_basic_environment
3050     )
3051 }
3052
3053 test_setup() {
3054     if get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3055             command -v meson >/dev/null && \
3056             [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3057         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3058         exit 1
3059     fi
3060
3061     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3062         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3063         mount_initdir
3064     else
3065         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3066             # default.img is the base that every test uses and optionally appends to
3067             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3068                 # Create the backing public image, but then completely unmount
3069                 # it and drop the loopback device responsible for it, since we're
3070                 # going to symlink/copy the image and mount it again from
3071                 # elsewhere.
3072                 local image_old="${IMAGE_PUBLIC}"
3073                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3074                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3075                 fi
3076                 test_create_image
3077                 test_setup_cleanup
3078                 umount_loopback
3079                 cleanup_loopdev
3080                 IMAGE_PUBLIC="${image_old}"
3081             fi
3082             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3083                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3084             fi
3085         fi
3086
3087         local hook_defined
3088         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3089
3090         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3091         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3092             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3093         else
3094             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3095         fi
3096
3097         mount_initdir
3098
3099         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3100             dinfo "Masking supporting services"
3101             mask_supporting_services
3102         fi
3103
3104         # Send stdout/stderr of testsuite-*.service units to both journal and
3105         # console to make debugging in CIs easier
3106         # Note: we can't use a dropin for `testsuite-.service`, since that also
3107         #       overrides 'sub-units' of some tests that already use a specific
3108         #       value for Standard(Output|Error)=
3109         #       (e.g. test/units/testsuite-66-deviceisolation.service)
3110         if ! get_bool "$INTERACTIVE_DEBUG"; then
3111             local dropin_dir="${initdir:?}/etc/systemd/system/testsuite-${TESTID:?}.service.d"
3112             mkdir -p "$dropin_dir"
3113             printf '[Service]\nStandardOutput=journal+console\nStandardError=journal+console' >"$dropin_dir/99-stdout.conf"
3114         fi
3115
3116         if get_bool "$hook_defined"; then
3117             test_append_files "${initdir:?}"
3118         fi
3119     fi
3120
3121     setup_nspawn_root
3122 }
3123
3124 test_run() {
3125     local test_id="${1:?}"
3126     mount_initdir
3127
3128     if ! get_bool "${TEST_NO_QEMU:=}"; then
3129         if run_qemu "$test_id"; then
3130             check_result_qemu || { echo "qemu test failed"; return 1; }
3131         else
3132             dwarn "can't run qemu, skipping"
3133         fi
3134     fi
3135     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3136         mount_initdir
3137         if run_nspawn "${initdir:?}" "$test_id"; then
3138             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3139         else
3140             dwarn "can't run systemd-nspawn, skipping"
3141         fi
3142
3143         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3144             dir="$TESTDIR/unprivileged-nspawn-root"
3145             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3146                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3147             else
3148                 dwarn "can't run systemd-nspawn, skipping"
3149             fi
3150         fi
3151     fi
3152     return 0
3153 }
3154
3155 do_test() {
3156     if [[ $UID != "0" ]]; then
3157         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3158         exit 0
3159     fi
3160
3161     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3162         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3163         exit 0
3164     fi
3165
3166     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3167         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3168         exit 0
3169     fi
3170
3171     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3172         TEST_NO_QEMU=1
3173     fi
3174
3175     # Detect lib paths
3176     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3177         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3178     done
3179
3180     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3181         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3182     done
3183
3184     mkdir -p "$STATEDIR"
3185
3186     import_testdir
3187     import_initdir
3188
3189     if [ -n "${SUDO_USER}" ]; then
3190         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3191         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3192     fi
3193
3194     testname="$(basename "$PWD")"
3195
3196     while (($# > 0)); do
3197         case $1 in
3198             --run)
3199                 echo "${testname} RUN: $TEST_DESCRIPTION"
3200                 test_run "$TESTID"
3201                 ret=$?
3202                 if [ $ret -eq 0 ]; then
3203                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3204                 else
3205                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3206                 fi
3207                 exit $ret
3208                 ;;
3209             --setup)
3210                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3211                 test_setup
3212                 test_setup_cleanup
3213                 ;;
3214             --clean)
3215                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3216                 test_cleanup
3217                 ;;
3218             --clean-again)
3219                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3220                 test_cleanup_again
3221                 ;;
3222             --all)
3223                 ret=0
3224                 echo -n "${testname}: $TEST_DESCRIPTION "
3225                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3226                 # and loop devices will leak
3227                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3228                 if [ $ret -eq 0 ]; then
3229                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3230                 fi
3231                 if [ $ret -eq 0 ]; then
3232                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3233                 fi
3234                 test_cleanup
3235                 if [ $ret -eq 0 ]; then
3236                     rm "$TESTLOG"
3237                     echo "[OK]"
3238                 else
3239                     echo "[FAILED]"
3240                     echo "see $TESTLOG"
3241                 fi
3242                 exit $ret
3243                 ;;
3244             *)
3245                 break
3246                 ;;
3247         esac
3248         shift
3249     done
3250 }