test/test-functions

   1 #!/usr/bin/env bash
   2 # -*- mode: shell-script; indent-tabs-mode: nil; sh-basic-offset: 4; -*-
   3 # SPDX-License-Identifier: LGPL-2.1-or-later
   4 #
   5 # shellcheck disable=SC2030,SC2031
   6 # ex: ts=8 sw=4 sts=4 et filetype=sh tw=180
   7 # Note: the shellcheck line above disables warning for variables which were
   8 #       modified in a subshell. In our case this behavior is expected, but
   9 #       `shellcheck` can't distinguish this because of poor variable tracking,
  10 #       which results in warning for every instance of such variable used
  11 #       throughout this file.
  12 #       See:
  13 #           * comment in function install_verity_minimal()
  14 #           * koalaman/shellcheck#280
  15 set -o pipefail
  16
  17 PATH=/sbin:/bin:/usr/sbin:/usr/bin
  18 export PATH
  19
  20 os_release=$(test -e /etc/os-release && echo /etc/os-release || echo /usr/lib/os-release)
  21 # shellcheck source=/dev/null
  22 source "$os_release"
  23 [[ "$ID" = "debian" || " $ID_LIKE " = *" debian "* ]] && LOOKS_LIKE_DEBIAN=yes || LOOKS_LIKE_DEBIAN=""
  24 [[ "$ID" = "arch" || " $ID_LIKE " = *" arch "* ]] && LOOKS_LIKE_ARCH=yes || LOOKS_LIKE_ARCH=""
  25 [[ " $ID_LIKE " = *" suse "* ]] && LOOKS_LIKE_SUSE=yes || LOOKS_LIKE_SUSE=""
  26 KERNEL_VER="${KERNEL_VER-$(uname -r)}"
  27 QEMU_TIMEOUT="${QEMU_TIMEOUT:-1800}"
  28 NSPAWN_TIMEOUT="${NSPAWN_TIMEOUT:-1800}"
  29 TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
  30 [[ "$LOOKS_LIKE_SUSE" ]] && FSTYPE="${FSTYPE:-btrfs}" || FSTYPE="${FSTYPE:-ext4}"
  31 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
  32 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
  33 # Note that defining a different IMAGE_NAME in a test setup script will only result
  34 # in default.img being copied and renamed. It can then be extended by defining
  35 # a test_append_files() function. The $1 parameter will be the root directory.
  36 # To force creating a new image from scratch (eg: to encrypt it), also define
  37 # TEST_FORCE_NEWIMAGE=1 in the test setup script.
  38 IMAGE_NAME=${IMAGE_NAME:-default}
  39 TEST_REQUIRE_INSTALL_TESTS="${TEST_REQUIRE_INSTALL_TESTS:-1}"
  40 TEST_PARALLELIZE="${TEST_PARALLELIZE:-0}"
  41 TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED="${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED:-1}"
  42 LOOPDEV=
  43
  44 # Simple wrapper to unify boolean checks.
  45 # Note: this function needs to stay near the top of the file, so we can use it
  46 #       in code in the outermost scope.
  47 get_bool() {
  48     # Make the value lowercase to make the regex matching simpler
  49     local _bool="${1,,}"
  50
  51     # Consider empty value as "false"
  52     if [[ -z "$_bool" || "$_bool" =~ ^(0|no|false)$ ]]; then
  53         return 1
  54     elif [[ "$_bool" =~ ^(1|yes|true)$ ]]; then
  55         return 0
  56     else
  57         echo >&2 "Value '$_bool' is not a valid boolean value"
  58         exit 1
  59     fi
  60 }
  61
  62 # Since in Bash we can have only one handler per signal, let's overcome this
  63 # limitation by having one global handler for the EXIT signal which executes
  64 # all registered handlers
  65 _AT_EXIT_HANDLERS=()
  66 _at_exit() {
  67     set +e
  68
  69     # Run the EXIT handlers in reverse order
  70     for ((i = ${#_AT_EXIT_HANDLERS[@]} - 1; i >= 0; i--)); do
  71         ddebug "Running EXIT handler '${_AT_EXIT_HANDLERS[$i]}'"
  72         "${_AT_EXIT_HANDLERS[$i]}"
  73     done
  74 }
  75
  76 trap _at_exit EXIT
  77
  78 add_at_exit_handler() {
  79     local handler="${1?}"
  80
  81     if [[ "$(type -t "$handler")" != "function" ]]; then
  82         dfatal "'$handler' is not a function"
  83         exit 1
  84     fi
  85
  86     _AT_EXIT_HANDLERS+=("$handler")
  87 }
  88
  89 # Decide if we can (and want to) run qemu with KVM acceleration.
  90 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
  91 # check if it's not explicitly disabled (TEST_NO_KVM) and we're not already
  92 # running under KVM. If these conditions are met, enable KVM (and possibly
  93 # nested KVM), otherwise disable it.
  94 if get_bool "${TEST_NESTED_KVM:=}" || (! get_bool "${TEST_NO_KVM:=}" && ! systemd-detect-virt -qv); then
  95     QEMU_KVM=yes
  96 else
  97     QEMU_KVM=no
  98 fi
  99
 100 if ! ROOTLIBDIR=$(pkg-config --variable=systemdutildir systemd); then
 101     echo "WARNING! Cannot determine rootlibdir from pkg-config, assuming /usr/lib/systemd" >&2
 102     ROOTLIBDIR=/usr/lib/systemd
 103 fi
 104
 105 # The calling test.sh scripts have TEST_BASE_DIR set via their Makefile, but we don't need them to provide it
 106 TEST_BASE_DIR=${TEST_BASE_DIR:-$(realpath "$(dirname "${BASH_SOURCE[0]}")")}
 107 TEST_UNITS_DIR="$(realpath "$TEST_BASE_DIR/units")"
 108 SOURCE_DIR=$(realpath "$TEST_BASE_DIR/..")
 109 # These variables are used by test scripts
 110 export TEST_BASE_DIR TEST_UNITS_DIR SOURCE_DIR
 111
 112 TOOLS_DIR="$SOURCE_DIR/tools"
 113 # note that find-build-dir.sh will return $BUILD_DIR if provided, else it will try to find it
 114 if get_bool "${NO_BUILD:=}"; then
 115     BUILD_DIR="$SOURCE_DIR"
 116 elif ! BUILD_DIR="$("$TOOLS_DIR"/find-build-dir.sh)"; then
 117     echo "ERROR: no build found, please set BUILD_DIR or use NO_BUILD" >&2
 118     exit 1
 119 fi
 120
 121 PATH_TO_INIT="$ROOTLIBDIR/systemd"
 122 SYSTEMD_JOURNALD="${SYSTEMD_JOURNALD:-$(command -v "$BUILD_DIR/systemd-journald" || command -v "$ROOTLIBDIR/systemd-journald")}"
 123 SYSTEMD_JOURNAL_REMOTE="${SYSTEMD_JOURNAL_REMOTE:-$(command -v "$BUILD_DIR/systemd-journal-remote" || command -v "$ROOTLIBDIR/systemd-journal-remote" || echo "")}"
 124 SYSTEMD="${SYSTEMD:-$(command -v "$BUILD_DIR/systemd" || command -v "$ROOTLIBDIR/systemd")}"
 125 SYSTEMD_NSPAWN="${SYSTEMD_NSPAWN:-$(command -v "$BUILD_DIR/systemd-nspawn" || command -v systemd-nspawn)}"
 126 JOURNALCTL="${JOURNALCTL:-$(command -v "$BUILD_DIR/journalctl" || command -v journalctl)}"
 127 SYSTEMCTL="${SYSTEMCTL:-$(command -v "$BUILD_DIR/systemctl" || command -v systemctl)}"
 128
 129 TESTFILE="${BASH_SOURCE[1]}"
 130 if [ -z "$TESTFILE" ]; then
 131     echo "ERROR: test-functions must be sourced from one of the TEST-*/test.sh scripts" >&2
 132     exit 1
 133 fi
 134 TESTNAME="$(basename "$(dirname "$(realpath "$TESTFILE")")")"
 135
 136 WORKDIR="/var/tmp/systemd-tests"
 137 if get_bool "${NO_BUILD:=}"; then
 138     STATEDIR="$WORKDIR/$TESTNAME"
 139 else
 140     STATEDIR="$BUILD_DIR/test/$TESTNAME"
 141 fi
 142
 143 STATEFILE="$STATEDIR/.testdir"
 144 IMAGESTATEDIR="$STATEDIR/.."
 145 TESTLOG="$STATEDIR/test.log"
 146
 147 if ! [[ "$TESTNAME" =~ ^TEST\-([0-9]+)\-.+$ ]]; then
 148     echo "ERROR: Test name '$TESTNAME' is not in the expected format: TEST-[0-9]+-*" >&2
 149     exit 1
 150 fi
 151 TESTID="${BASH_REMATCH[1]:?}"
 152
 153 if [[ ! -f "$TEST_UNITS_DIR/testsuite-$TESTID.service" ]]; then
 154     echo "ERROR: Test '$TESTNAME' is missing its service file '$TEST_UNITS_DIR/testsuite-$TESTID.service" >&2
 155     exit 1
 156 fi
 157
 158 BASICTOOLS=(
 159     awk
 160     base64
 161     basename
 162     bash
 163     capsh
 164     cat
 165     chmod
 166     chown
 167     chroot
 168     cmp
 169     cryptsetup
 170     cut
 171     date
 172     dd
 173     diff
 174     dirname
 175     dmsetup
 176     echo
 177     env
 178     false
 179     flock
 180     getconf
 181     getent
 182     getfacl
 183     grep
 184     gunzip
 185     gzip
 186     head
 187     ionice
 188     ip
 189     jq
 190     killall
 191     ldd
 192     ln
 193     loadkeys
 194     login
 195     losetup
 196     lsattr
 197     lz4cat
 198     mkfifo
 199     mknod
 200     mktemp
 201     modprobe
 202     mount
 203     mountpoint
 204     mv
 205     nc
 206     nproc
 207     pkill
 208     readlink
 209     realpath
 210     rev
 211     rm
 212     rmdir
 213     rmmod
 214     script
 215     sed
 216     seq
 217     setfacl
 218     setfattr
 219     setfont
 220     setpriv
 221     setsid
 222     sfdisk
 223     sh
 224     sleep
 225     stat
 226     stty
 227     su
 228     sulogin
 229     sysctl
 230     tail
 231     tar
 232     tee
 233     test
 234     timeout
 235     touch
 236     tr
 237     true
 238     truncate
 239     umount
 240     uname
 241     unshare
 242     useradd
 243     userdel
 244     wc
 245     xargs
 246     xzcat
 247 )
 248
 249 DEBUGTOOLS=(
 250     cp
 251     df
 252     dhclient
 253     dmesg
 254     du
 255     find
 256     free
 257     grep
 258     hostname
 259     id
 260     less
 261     ln
 262     ls
 263     mkdir
 264     ping
 265     ps
 266     route
 267     sort
 268     strace
 269     tty
 270     vi
 271     /usr/libexec/vi
 272 )
 273
 274 is_built_with_asan() {
 275     local _bin="${1:?}"
 276
 277     if ! type -P objdump >/dev/null; then
 278         ddebug "Failed to find objdump. Assuming systemd hasn't been built with ASAN."
 279         return 1
 280     fi
 281
 282     # Borrowed from https://github.com/google/oss-fuzz/blob/cd9acd02f9d3f6e80011cc1e9549be526ce5f270/infra/base-images/base-runner/bad_build_check#L182
 283     local _asan_calls
 284     _asan_calls="$(objdump -dC "$_bin" | grep -E "(callq?|brasl?|bl)\s.+__asan" -c)"
 285     if ((_asan_calls < 1000)); then
 286         return 1
 287     else
 288         return 0
 289     fi
 290 }
 291
 292 is_built_with_coverage() {
 293     if get_bool "${NO_BUILD:=}" || ! command -v meson >/dev/null; then
 294         return 1
 295     fi
 296
 297     meson configure "${BUILD_DIR:?}" | grep 'b_coverage' | awk '{ print $2 }' | grep -q 'true'
 298 }
 299
 300 IS_BUILT_WITH_ASAN=$(is_built_with_asan "$SYSTEMD_JOURNALD" && echo yes || echo no)
 301 IS_BUILT_WITH_COVERAGE=$(is_built_with_coverage && echo yes || echo no)
 302
 303 if get_bool "$IS_BUILT_WITH_ASAN"; then
 304     SKIP_INITRD="${SKIP_INITRD:-yes}"
 305     PATH_TO_INIT=$ROOTLIBDIR/systemd-under-asan
 306     QEMU_MEM="${QEMU_MEM:-2G}"
 307     QEMU_SMP="${QEMU_SMP:-4}"
 308
 309     # We need to correctly distinguish between gcc's and clang's ASan DSOs.
 310     if ASAN_RT_NAME="$(awk '/libasan.so/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 311         ASAN_COMPILER=gcc
 312         ASAN_RT_PATH="$(readlink -f "$(${CC:-gcc} --print-file-name "$ASAN_RT_NAME")")"
 313     elif ASAN_RT_NAME="$(awk '/libclang_rt.asan/ {x=$1; exit} END {print x; exit x==""}' < <(ldd "$SYSTEMD"))"; then
 314         ASAN_COMPILER=clang
 315         ASAN_RT_PATH="$(readlink -f "$(${CC:-clang} --print-file-name "$ASAN_RT_NAME")")"
 316
 317         # As clang's ASan DSO is usually in a non-standard path, let's check if
 318         # the environment is set accordingly. If not, warn the user and exit.
 319         # We're not setting the LD_LIBRARY_PATH automagically here, because
 320         # user should encounter (and fix) the same issue when running the unit
 321         # tests (meson test)
 322         if ldd "$SYSTEMD" | grep -q "libclang_rt.asan.*not found"; then
 323             echo >&2 "clang's ASan DSO ($ASAN_RT_NAME) is not present in the runtime library path"
 324             echo >&2 "Consider setting LD_LIBRARY_PATH=${ASAN_RT_PATH%/*}"
 325             exit 1
 326         fi
 327     else
 328         echo >&2 "systemd is not linked against the ASan DSO"
 329         echo >&2 "gcc does this by default, for clang compile with -shared-libasan"
 330         exit 1
 331     fi
 332
 333     echo "Detected ASan RT '$ASAN_RT_NAME' located at '$ASAN_RT_PATH'"
 334 fi
 335
 336 find_qemu_bin() {
 337     QEMU_BIN="${QEMU_BIN:-""}"
 338     # SUSE and Red Hat call the binary qemu-kvm. Debian and Gentoo call it kvm.
 339     if get_bool "$QEMU_KVM"; then
 340         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v kvm qemu-kvm 2>/dev/null | grep '^/' -m1)"
 341     fi
 342
 343     [[ -n "$ARCH" ]] || ARCH="$(uname -m)"
 344     case $ARCH in
 345     x86_64)
 346         # QEMU's own build system calls it qemu-system-x86_64
 347         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-x86_64 2>/dev/null | grep '^/' -m1)"
 348         ;;
 349     i*86)
 350         # new i386 version of QEMU
 351         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-i386 2>/dev/null | grep '^/' -m1)"
 352
 353         # i386 version of QEMU
 354         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu 2>/dev/null | grep '^/' -m1)"
 355         ;;
 356     ppc64*)
 357         [[ -n "$QEMU_BIN" ]] || QEMU_BIN="$(command -v qemu-system-ppc64 2>/dev/null | grep '^/' -m1)"
 358         ;;
 359     esac
 360
 361     if [[ ! -e "$QEMU_BIN" ]]; then
 362         echo "Could not find a suitable qemu binary" >&2
 363         return 1
 364     fi
 365 }
 366
 367 # Compares argument #1=X.Y.Z (X&Y&Z = numeric) to the version of the installed qemu
 368 # returns 0 if newer or equal
 369 # returns 1 if older
 370 # returns 2 if failing
 371 qemu_min_version() {
 372     find_qemu_bin || return 2
 373
 374     # get version from binary
 375     local qemu_ver
 376     qemu_ver="$("$QEMU_BIN" --version | awk '/^QEMU emulator version ([0-9]*\.[0-9]*\.[0-9]*)/ {print $4}')"
 377
 378     # Check version string format
 379     echo "$qemu_ver" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 380     echo "$1" | grep -q '^[0-9]*\.[0-9]*\.[0-9]*$' || return 2
 381
 382     # compare as last command to return that value
 383     printf "%s\n%s\n" "$1" "$qemu_ver" | sort -V -C
 384 }
 385
 386 # Return 0 if qemu did run (then you must check the result state/logs for actual
 387 # success), or 1 if qemu is not available.
 388 run_qemu() {
 389     # If the test provided its own initrd, use it (e.g. TEST-24)
 390     if [[ -z "$INITRD" && -f "${TESTDIR:?}/initrd.img" ]]; then
 391         INITRD="$TESTDIR/initrd.img"
 392     fi
 393
 394     if [ -f /etc/machine-id ]; then
 395         read -r MACHINE_ID </etc/machine-id
 396         [ -z "$INITRD" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd" ] \
 397             && INITRD="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/initrd"
 398         [ -z "$KERNEL_BIN" ] && [ -e "$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux" ] \
 399             && KERNEL_BIN="$EFI_MOUNT/$MACHINE_ID/$KERNEL_VER/linux"
 400     fi
 401
 402     local CONSOLE=ttyS0
 403
 404     rm -f "$initdir"/{testok,failed,skipped}
 405     # make sure the initdir is not mounted to avoid concurrent access
 406     cleanup_initdir
 407     umount_loopback
 408
 409     if [[ ! "$KERNEL_BIN" ]]; then
 410         if get_bool "$LOOKS_LIKE_ARCH"; then
 411             KERNEL_BIN=/boot/vmlinuz-linux
 412         else
 413             [ "$ARCH" ] || ARCH=$(uname -m)
 414             case $ARCH in
 415                 ppc64*)
 416                     # Ubuntu ppc64* calls the kernel binary as vmlinux-*, RHEL/CentOS
 417                     # uses the "standard" vmlinuz- prefix
 418                     [[ -e "/boot/vmlinux-$KERNEL_VER" ]] && KERNEL_BIN="/boot/vmlinux-$KERNEL_VER" || KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 419                     CONSOLE=hvc0
 420                     ;;
 421                 *)
 422                     KERNEL_BIN="/boot/vmlinuz-$KERNEL_VER"
 423                     ;;
 424             esac
 425         fi
 426     fi
 427
 428     local default_fedora_initrd="/boot/initramfs-${KERNEL_VER}.img"
 429     local default_debian_initrd="/boot/initrd.img-${KERNEL_VER}"
 430     local default_arch_initrd="/boot/initramfs-linux-fallback.img"
 431     local default_suse_initrd="/boot/initrd-${KERNEL_VER}"
 432     if [[ ! "$INITRD" ]]; then
 433         if [[ -e "$default_fedora_initrd" ]]; then
 434             INITRD="$default_fedora_initrd"
 435         elif [[ "$LOOKS_LIKE_DEBIAN" && -e "$default_debian_initrd" ]]; then
 436             INITRD="$default_debian_initrd"
 437         elif [[ "$LOOKS_LIKE_ARCH" && -e "$default_arch_initrd" ]]; then
 438             INITRD="$default_arch_initrd"
 439         elif [[ "$LOOKS_LIKE_SUSE" && -e "$default_suse_initrd" ]]; then
 440             INITRD="$default_suse_initrd"
 441         fi
 442     fi
 443
 444     # If QEMU_SMP was not explicitly set, try to determine the value 'dynamically'
 445     # i.e. use the number of online CPUs on the host machine. If the nproc utility
 446     # is not installed or there's some other error when calling it, fall back
 447     # to the original value (QEMU_SMP=1).
 448     if [[ -z "${QEMU_SMP:=}" ]]; then
 449         if ! QEMU_SMP=$(nproc); then
 450             dwarn "nproc utility is not installed, falling back to QEMU_SMP=1"
 451             QEMU_SMP=1
 452         fi
 453     fi
 454
 455     find_qemu_bin || return 1
 456
 457     # Umount initdir to avoid concurrent access to the filesystem
 458     _umount_dir "$initdir"
 459
 460     local kernel_params=()
 461     local qemu_options=()
 462     local qemu_cmd=("$QEMU_BIN")
 463
 464     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
 465         kernel_params+=("systemd.unified_cgroup_hierarchy=yes")
 466     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 467         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=yes")
 468     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 469         kernel_params+=("systemd.unified_cgroup_hierarchy=no" "systemd.legacy_systemd_cgroup_controller=no")
 470     elif [[ "$UNIFIED_CGROUP_HIERARCHY" != "default" ]]; then
 471         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 472         exit 1
 473     fi
 474
 475     if get_bool "$LOOKS_LIKE_SUSE"; then
 476         kernel_params+=("rd.hostonly=0")
 477     fi
 478
 479     # Debian/Ubuntu's initramfs tries to check if it can resume from hibernation
 480     # and wastes a minute or so probing disks, skip that as it's not useful here
 481     kernel_params+=(
 482         "root=LABEL=systemd_boot"
 483         "rw"
 484         "raid=noautodetect"
 485         "rd.luks=0"
 486         "loglevel=2"
 487         "init=$PATH_TO_INIT"
 488         "console=$CONSOLE"
 489         "SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$1.units:/usr/lib/systemd/tests/testdata/units:"
 490         "systemd.unit=testsuite.target"
 491         "systemd.wants=testsuite-$1.service"
 492         "noresume"
 493     )
 494
 495     if ! get_bool "$INTERACTIVE_DEBUG"; then
 496         kernel_params+=(
 497             "oops=panic"
 498             "panic=1"
 499             "softlockup_panic=1"
 500             "systemd.wants=end.service"
 501         )
 502     fi
 503
 504     [ -e "$IMAGE_PRIVATE" ] && image="$IMAGE_PRIVATE" || image="$IMAGE_PUBLIC"
 505     qemu_options+=(
 506         -smp "$QEMU_SMP"
 507         -net none
 508         -m "${QEMU_MEM:-768M}"
 509         -nographic
 510         -kernel "$KERNEL_BIN"
 511         -drive "format=raw,cache=unsafe,file=$image"
 512         -device "virtio-rng-pci,max-bytes=1024,period=1000"
 513     )
 514
 515     if [[ -n "${QEMU_OPTIONS:=}" ]]; then
 516         local user_qemu_options
 517         read -ra user_qemu_options <<< "$QEMU_OPTIONS"
 518         qemu_options+=("${user_qemu_options[@]}")
 519     fi
 520     qemu_options+=(${QEMU_OPTIONS_ARRAY:+"${QEMU_OPTIONS_ARRAY[@]}"})
 521
 522     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 523         local user_kernel_append
 524         readarray user_kernel_append <<< "$KERNEL_APPEND"
 525         kernel_params+=("${user_kernel_append[@]}")
 526     fi
 527
 528     if [[ "$INITRD" ]] && ! get_bool "$SKIP_INITRD"; then
 529         qemu_options+=(-initrd "$INITRD")
 530     fi
 531
 532     # Let's use KVM if possible
 533     if [[ -c /dev/kvm ]] && get_bool $QEMU_KVM; then
 534         qemu_options+=(-machine "accel=kvm" -enable-kvm -cpu host)
 535     fi
 536
 537     if [[ "$QEMU_TIMEOUT" != "infinity" ]]; then
 538         qemu_cmd=(timeout --foreground "$QEMU_TIMEOUT" "$QEMU_BIN")
 539     fi
 540
 541     (set -x; "${qemu_cmd[@]}" "${qemu_options[@]}" -append "${kernel_params[*]}")
 542     rc=$?
 543     if [ "$rc" -eq 124 ] && [ "$QEMU_TIMEOUT" != "infinity" ]; then
 544         derror "Test timed out after ${QEMU_TIMEOUT}s"
 545         TIMED_OUT=1
 546     else
 547         [ "$rc" != 0 ] && derror "qemu failed with exit code $rc"
 548     fi
 549     return 0
 550 }
 551
 552 # Return 0 if nspawn did run (then you must check the result state/logs for actual
 553 # success), or 1 if nspawn is not available.
 554 run_nspawn() {
 555     [[ -d /run/systemd/system ]] || return 1
 556     rm -f "${initdir:?}"/{testok,failed,skipped}
 557
 558     local nspawn_cmd=()
 559     local nspawn_options=(
 560         "--register=no"
 561         "--kill-signal=SIGKILL"
 562         "--directory=${1:?}"
 563         "--setenv=SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/testsuite-$2.units:/usr/lib/systemd/tests/testdata/units:"
 564         "--machine=TEST-$TESTID"
 565     )
 566     local kernel_params=(
 567         "$PATH_TO_INIT"
 568         "systemd.unit=testsuite.target"
 569         "systemd.wants=testsuite-$2.service"
 570     )
 571
 572     if ! get_bool "$INTERACTIVE_DEBUG"; then
 573         kernel_params+=("systemd.wants=end.service")
 574     fi
 575
 576     if [[ -n "${NSPAWN_ARGUMENTS:=}" ]]; then
 577         local user_nspawn_arguments
 578         read -ra user_nspawn_arguments <<< "$NSPAWN_ARGUMENTS"
 579         nspawn_options+=("${user_nspawn_arguments[@]}")
 580     fi
 581
 582     if [[ -n "${KERNEL_APPEND:=}" ]]; then
 583         local user_kernel_append
 584         readarray user_kernel_append <<< "$KERNEL_APPEND"
 585         kernel_params+=("${user_kernel_append[@]}")
 586     fi
 587
 588     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "hybrid" ]]; then
 589         dwarn "nspawn doesn't support SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=hybrid, skipping"
 590         exit
 591     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" || "$UNIFIED_CGROUP_HIERARCHY" = "no" ]]; then
 592         nspawn_cmd+=(env "SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=$UNIFIED_CGROUP_HIERARCHY")
 593     elif [[ "$UNIFIED_CGROUP_HIERARCHY" = "default" ]]; then
 594         nspawn_cmd+=(env "--unset=UNIFIED_CGROUP_HIERARCHY" "--unset=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY")
 595     else
 596         dfatal "Unknown UNIFIED_CGROUP_HIERARCHY. Got $UNIFIED_CGROUP_HIERARCHY, expected [yes|no|hybrid|default]"
 597         exit 1
 598     fi
 599
 600     if [[ "$NSPAWN_TIMEOUT" != "infinity" ]]; then
 601         nspawn_cmd+=(timeout --foreground "$NSPAWN_TIMEOUT" "$SYSTEMD_NSPAWN")
 602     else
 603         nspawn_cmd+=("$SYSTEMD_NSPAWN")
 604     fi
 605
 606     (set -x; "${nspawn_cmd[@]}" "${nspawn_options[@]}" "${kernel_params[@]}")
 607     rc=$?
 608     if [ "$rc" -eq 124 ] && [ "$NSPAWN_TIMEOUT" != "infinity" ]; then
 609         derror "Test timed out after ${NSPAWN_TIMEOUT}s"
 610         TIMED_OUT=1
 611     else
 612         [ "$rc" != 0 ] && derror "nspawn failed with exit code $rc"
 613     fi
 614     return 0
 615 }
 616
 617 # Build two very minimal root images, with two units, one is the same and one is different across them
 618 install_verity_minimal() {
 619     dinfo "Set up a set of minimal images for verity verification"
 620     if [ -e "$initdir/usr/share/minimal.raw" ]; then
 621         return
 622     fi
 623     if ! command -v mksquashfs >/dev/null 2>&1; then
 624         dfatal "mksquashfs not found"
 625         exit 1
 626     fi
 627     if ! command -v veritysetup >/dev/null 2>&1; then
 628         dfatal "veritysetup not found"
 629         exit 1
 630     fi
 631     # Local modifications of some global variables is intentional in this
 632     # subshell (SC2030)
 633     # shellcheck disable=SC2030
 634     (
 635         BASICTOOLS=(
 636             bash
 637             cat
 638             grep
 639             mount
 640             sleep
 641         )
 642         oldinitdir="$initdir"
 643         rm -rfv "$TESTDIR/minimal"
 644         export initdir="$TESTDIR/minimal"
 645         # app0 will use TemporaryFileSystem=/var/lib, app1 will need the mount point in the base image
 646         mkdir -p "$initdir/usr/lib/systemd/system" "$initdir/usr/lib/extension-release.d" "$initdir/etc" "$initdir/var/tmp" "$initdir/opt" "$initdir/var/lib/app1"
 647         setup_basic_dirs
 648         install_basic_tools
 649         install_ld_so_conf
 650         # Shellcheck treats [[ -v VAR ]] as an assignment to avoid a different
 651         # issue, thus falsely triggering SC2030 in this case
 652         # See: koalaman/shellcheck#1409
 653         if [[ -v ASAN_RT_PATH ]]; then
 654             # If we're compiled with ASan, install the ASan RT (and its dependencies)
 655             # into the verity images to get rid of the annoying errors about
 656             # missing $LD_PRELOAD libraries.
 657             inst_libs "$ASAN_RT_PATH"
 658             inst_library "$ASAN_RT_PATH"
 659         fi
 660         cp "$os_release" "$initdir/usr/lib/os-release"
 661         ln -s ../usr/lib/os-release "$initdir/etc/os-release"
 662         touch "$initdir/etc/machine-id" "$initdir/etc/resolv.conf"
 663         touch "$initdir/opt/some_file"
 664         echo MARKER=1 >>"$initdir/usr/lib/os-release"
 665         echo "PORTABLE_PREFIXES=app0 minimal minimal-app0" >>"$initdir/usr/lib/os-release"
 666         cat >"$initdir/usr/lib/systemd/system/minimal-app0.service" <<EOF
 667 [Service]
 668 ExecStartPre=cat /usr/lib/os-release
 669 ExecStart=sleep 120
 670 EOF
 671         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 672
 673         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_0.raw" -noappend
 674         veritysetup format "$oldinitdir/usr/share/minimal_0.raw" "$oldinitdir/usr/share/minimal_0.verity" | \
 675             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_0.roothash"
 676
 677         sed -i "s/MARKER=1/MARKER=2/g" "$initdir/usr/lib/os-release"
 678         rm "$initdir/usr/lib/systemd/system/minimal-app0-foo.service"
 679         cp "$initdir/usr/lib/systemd/system/minimal-app0.service" "$initdir/usr/lib/systemd/system/minimal-app0-bar.service"
 680
 681         mksquashfs "$initdir" "$oldinitdir/usr/share/minimal_1.raw" -noappend
 682         veritysetup format "$oldinitdir/usr/share/minimal_1.raw" "$oldinitdir/usr/share/minimal_1.verity" | \
 683             grep '^Root hash:' | cut -f2 | tr -d '\n' >"$oldinitdir/usr/share/minimal_1.roothash"
 684
 685         # Rolling distros like Arch do not set VERSION_ID
 686         local version_id=""
 687         if grep -q "^VERSION_ID=" "$os_release"; then
 688             version_id="$(grep "^VERSION_ID=" "$os_release")"
 689         fi
 690
 691         export initdir="$TESTDIR/app0"
 692         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 693         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app0"
 694         echo "${version_id}" >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 695         ( echo "${version_id}"
 696           echo "SYSEXT_IMAGE_ID=app" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app0"
 697         cat >"$initdir/usr/lib/systemd/system/app0.service" <<EOF
 698 [Service]
 699 Type=oneshot
 700 RemainAfterExit=yes
 701 ExecStart=/opt/script0.sh
 702 TemporaryFileSystem=/var/lib
 703 StateDirectory=app0
 704 RuntimeDirectory=app0
 705 EOF
 706         cat >"$initdir/opt/script0.sh" <<EOF
 707 #!/bin/bash
 708 set -e
 709 test -e /usr/lib/os-release
 710 echo bar >\${STATE_DIRECTORY}/foo
 711 cat /usr/lib/extension-release.d/extension-release.app0
 712 EOF
 713         chmod +x "$initdir/opt/script0.sh"
 714         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 715         mksquashfs "$initdir" "$oldinitdir/usr/share/app0.raw" -noappend
 716
 717         export initdir="$TESTDIR/app1"
 718         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system" "$initdir/opt"
 719         grep "^ID=" "$os_release" >"$initdir/usr/lib/extension-release.d/extension-release.app2"
 720         ( echo "${version_id}"
 721           echo "SYSEXT_SCOPE=portable"
 722           echo "SYSEXT_IMAGE_ID=app"
 723           echo "SYSEXT_IMAGE_VERSION=1"
 724           echo "PORTABLE_PREFIXES=app1" ) >>"$initdir/usr/lib/extension-release.d/extension-release.app2"
 725         setfattr -n user.extension-release.strict -v false "$initdir/usr/lib/extension-release.d/extension-release.app2"
 726         cat >"$initdir/usr/lib/systemd/system/app1.service" <<EOF
 727 [Service]
 728 Type=oneshot
 729 RemainAfterExit=yes
 730 ExecStart=/opt/script1.sh
 731 StateDirectory=app1
 732 RuntimeDirectory=app1
 733 EOF
 734         cat >"$initdir/opt/script1.sh" <<EOF
 735 #!/bin/bash
 736 set -e
 737 test -e /usr/lib/os-release
 738 echo baz >\${STATE_DIRECTORY}/foo
 739 cat /usr/lib/extension-release.d/extension-release.app2
 740 EOF
 741         chmod +x "$initdir/opt/script1.sh"
 742         echo MARKER=1 >"$initdir/usr/lib/systemd/system/other_file"
 743         mksquashfs "$initdir" "$oldinitdir/usr/share/app1.raw" -noappend
 744
 745         export initdir="$TESTDIR/app-nodistro"
 746         mkdir -p "$initdir/usr/lib/extension-release.d" "$initdir/usr/lib/systemd/system"
 747         ( echo "ID=_any"
 748           echo "ARCHITECTURE=_any" ) >"$initdir/usr/lib/extension-release.d/extension-release.app-nodistro"
 749         echo MARKER=1 >"$initdir/usr/lib/systemd/system/some_file"
 750         mksquashfs "$initdir" "$oldinitdir/usr/share/app-nodistro.raw" -noappend
 751     )
 752 }
 753
 754 setup_basic_environment() {
 755     # create the basic filesystem layout
 756     setup_basic_dirs
 757
 758     install_systemd
 759     install_missing_libraries
 760     install_config_files
 761     install_zoneinfo
 762     create_rc_local
 763     install_basic_tools
 764     install_libnss
 765     install_pam
 766     install_dbus
 767     install_fonts
 768     install_locales
 769     install_keymaps
 770     install_x11_keymaps
 771     install_terminfo
 772     install_execs
 773     install_fs_tools
 774     install_modules
 775     install_plymouth
 776     install_haveged
 777     install_debug_tools
 778     install_ld_so_conf
 779     install_testuser
 780     has_user_dbus_socket && install_user_dbus
 781     setup_selinux
 782     install_depmod_files
 783     generate_module_dependencies
 784     if get_bool "$IS_BUILT_WITH_ASAN"; then
 785         create_asan_wrapper
 786     fi
 787     if get_bool "$TEST_INSTALL_VERITY_MINIMAL"; then
 788         install_verity_minimal
 789     fi
 790 }
 791
 792 setup_selinux() {
 793     dinfo "Setup SELinux"
 794     # don't forget KERNEL_APPEND='... selinux=1 ...'
 795     if ! get_bool "$SETUP_SELINUX"; then
 796         dinfo "SETUP_SELINUX != yes, skipping SELinux configuration"
 797         return 0
 798     fi
 799
 800     local conf_dir=/etc/selinux
 801     local fixfiles_tools=(bash uname cat sort uniq awk grep egrep head expr find rm secon setfiles)
 802
 803     # Make sure the following statement can't expand to "/" to prevent
 804     # a potential where-are-my-backups situation
 805     rm -rf "${initdir:?}/$conf_dir"
 806     if ! cp -ar "$conf_dir" "$initdir/$conf_dir"; then
 807         dfatal "Failed to copy $conf_dir"
 808         exit 1
 809     fi
 810
 811     touch "$initdir/.autorelabel"
 812     mkdir -p "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants"
 813     ln -sf ../autorelabel.service "$initdir/usr/lib/systemd/tests/testdata/units/basic.target.wants/"
 814
 815     image_install "${fixfiles_tools[@]}"
 816     image_install fixfiles sestatus
 817 }
 818
 819 install_valgrind() {
 820     if ! type -p valgrind; then
 821         dfatal "Failed to install valgrind"
 822         exit 1
 823     fi
 824
 825     local valgrind_bins valgrind_libs valgrind_supp
 826
 827     readarray -t valgrind_bins < <(strace -e execve valgrind /bin/true 2>&1 >/dev/null |
 828                                        sed -r -n 's/execve\("([^"]*)".*/\1/p')
 829     image_install "${valgrind_bins[@]}"
 830
 831     readarray -t valgrind_libs < <(LD_DEBUG=files valgrind /bin/true 2>&1 >/dev/null |
 832                                        sed -r -n 's|.*calling init: (/.*vgpreload_.*)|\1|p')
 833     image_install "${valgrind_libs[@]}"
 834
 835     readarray -t valgrind_supp < <(strace -e open valgrind /bin/true 2>&1 >/dev/null |
 836                                        sed -r -n 's,open\("([^"]*(/debug[^"]*|\.supp))".*= [0-9].*,\1,p')
 837     image_install "${valgrind_supp[@]}"
 838 }
 839
 840 create_valgrind_wrapper() {
 841     local valgrind_wrapper="$initdir/$ROOTLIBDIR/systemd-under-valgrind"
 842     ddebug "Create $valgrind_wrapper"
 843     cat >"$valgrind_wrapper" <<EOF
 844 #!/usr/bin/env bash
 845
 846 mount -t proc proc /proc
 847 exec valgrind --leak-check=full --track-fds=yes --log-file=/valgrind.out $ROOTLIBDIR/systemd "\$@"
 848 EOF
 849     chmod 0755 "$valgrind_wrapper"
 850 }
 851
 852 create_asan_wrapper() {
 853     local asan_wrapper="$initdir/$ROOTLIBDIR/systemd-under-asan"
 854     dinfo "Create ASan wrapper as '$asan_wrapper'"
 855
 856     [[ -z "$ASAN_RT_PATH" ]] && dfatal "ASAN_RT_PATH is empty, but it shouldn't be"
 857
 858     # clang: install llvm-symbolizer to generate useful reports
 859     # See: https://clang.llvm.org/docs/AddressSanitizer.html#symbolizing-the-reports
 860     [[ "$ASAN_COMPILER" == "clang" ]] && image_install "llvm-symbolizer"
 861
 862     cat >"$asan_wrapper" <<EOF
 863 #!/usr/bin/env bash
 864
 865 set -x
 866
 867 echo "ASan RT: $ASAN_RT_PATH"
 868 if [[ ! -e "$ASAN_RT_PATH" ]]; then
 869     echo >&2 "Couldn't find ASan RT at '$ASAN_RT_PATH', can't continue"
 870     exit 1
 871 fi
 872
 873 DEFAULT_ASAN_OPTIONS=${ASAN_OPTIONS:-strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1}
 874 DEFAULT_UBSAN_OPTIONS=${UBSAN_OPTIONS:-print_stacktrace=1:print_summary=1:halt_on_error=1}
 875 DEFAULT_ENVIRONMENT="ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS"
 876
 877 # Create a simple environment file which can be included by systemd services
 878 # that need it (i.e. services that utilize DynamicUser=true and bash, etc.)
 879 cat >/usr/lib/systemd/systemd-asan-env <<INNER_EOF
 880 LD_PRELOAD=$ASAN_RT_PATH
 881 ASAN_OPTIONS=$DEFAULT_ASAN_OPTIONS
 882 LSAN_OPTIONS=detect_leaks=0
 883 UBSAN_OPTIONS=$DEFAULT_UBSAN_OPTIONS
 884 INNER_EOF
 885
 886 # As right now bash is the PID 1, we can't expect PATH to have a sane value.
 887 # Let's make one to prevent unexpected "<bin> not found" issues in the future
 888 export PATH="/sbin:/bin:/usr/sbin:/usr/bin"
 889
 890 mountpoint -q /proc || mount -t proc proc /proc
 891 mountpoint -q /sys || mount -t sysfs sysfs /sys
 892 mount -o remount,rw /
 893
 894 DEFAULT_ENVIRONMENT="\$DEFAULT_ENVIRONMENT ASAN_RT_PATH=$ASAN_RT_PATH"
 895
 896 if [[ "$ASAN_COMPILER" == "clang" ]]; then
 897   # Let's add the ASan DSO's path to the dynamic linker's cache. This is pretty
 898   # unnecessary for gcc & libasan, however, for clang this is crucial, as its
 899   # runtime ASan DSO is in a non-standard (library) path.
 900   echo "${ASAN_RT_PATH%/*}" >/etc/ld.so.conf.d/asan-path-override.conf
 901   ldconfig
 902 fi
 903 echo DefaultEnvironment=\$DEFAULT_ENVIRONMENT >>/etc/systemd/system.conf
 904 echo DefaultTimeoutStartSec=180s >>/etc/systemd/system.conf
 905 echo DefaultStandardOutput=journal+console >>/etc/systemd/system.conf
 906
 907 # ASAN and syscall filters aren't compatible with each other.
 908 find / -name '*.service' -type f | xargs sed -i 's/^\\(MemoryDeny\\|SystemCall\\)/#\\1/'
 909
 910 # The redirection of ASAN reports to a file prevents them from ending up in /dev/null.
 911 # But, apparently, sometimes it doesn't work: https://github.com/google/sanitizers/issues/886.
 912 JOURNALD_CONF_DIR=/etc/systemd/system/systemd-journald.service.d
 913 mkdir -p "\$JOURNALD_CONF_DIR"
 914 printf "[Service]\nEnvironment=ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd-journald.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS:log_path=/systemd-journald.ubsan.log\n" >"\$JOURNALD_CONF_DIR/env.conf"
 915
 916 # Sometimes UBSan sends its reports to stderr regardless of what is specified in log_path
 917 # Let's try to catch them by redirecting stderr (and stdout just in case) to a file
 918 # See https://github.com/systemd/systemd/pull/12524#issuecomment-491108821
 919 printf "[Service]\nStandardOutput=file:/systemd-journald.out\n" >"\$JOURNALD_CONF_DIR/out.conf"
 920
 921 # 90s isn't enough for some services to finish when literally everything is run
 922 # under ASan+UBSan in containers, which, in turn, are run in VMs.
 923 # Let's limit which environments such services should be executed in.
 924 mkdir -p /etc/systemd/system/systemd-hwdb-update.service.d
 925 printf "[Unit]\nConditionVirtualization=container\n\n[Service]\nTimeoutSec=240s\n" >/etc/systemd/system/systemd-hwdb-update.service.d/env-override.conf
 926
 927 # Let's override another hard-coded timeout that kicks in too early
 928 mkdir -p /etc/systemd/system/systemd-journal-flush.service.d
 929 printf "[Service]\nTimeoutSec=180s\n" >/etc/systemd/system/systemd-journal-flush.service.d/timeout.conf
 930
 931 export ASAN_OPTIONS=\$DEFAULT_ASAN_OPTIONS:log_path=/systemd.asan.log UBSAN_OPTIONS=\$DEFAULT_UBSAN_OPTIONS
 932 exec "$ROOTLIBDIR/systemd" "\$@"
 933 EOF
 934
 935     chmod 0755 "$asan_wrapper"
 936 }
 937
 938 create_strace_wrapper() {
 939     local strace_wrapper="$initdir/$ROOTLIBDIR/systemd-under-strace"
 940     ddebug "Create $strace_wrapper"
 941     cat >"$strace_wrapper" <<EOF
 942 #!/usr/bin/env bash
 943
 944 exec strace -f -D -o /strace.out "$ROOTLIBDIR/systemd" "\$@"
 945 EOF
 946     chmod 0755 "$strace_wrapper"
 947 }
 948
 949 install_fs_tools() {
 950     dinfo "Install fsck"
 951     image_install /sbin/fsck*
 952     image_install -o /bin/fsck*
 953
 954     # fskc.reiserfs calls reiserfsck. so, install it
 955     image_install -o reiserfsck
 956
 957     # we use mkfs in system-repart tests
 958     image_install /sbin/mkfs.ext4
 959     image_install /sbin/mkfs.vfat
 960 }
 961
 962 install_modules() {
 963     dinfo "Install modules"
 964
 965     instmods bridge dummy ipvlan macvlan vfat veth
 966     instmods loop =block
 967     instmods nls_ascii =nls
 968     instmods overlay =overlayfs
 969     # for TEST-35-LOGIN
 970     instmods scsi_debug uinput
 971
 972     if get_bool "$LOOKS_LIKE_SUSE"; then
 973         instmods ext4
 974         # for TEST-54-CREDS
 975         instmods dmi-sysfs
 976     fi
 977 }
 978
 979 install_dmevent() {
 980     instmods dm_crypt =crypto
 981     inst_binary dmeventd
 982     image_install "${ROOTLIBDIR:?}"/system/dm-event.{service,socket}
 983     if get_bool "$LOOKS_LIKE_DEBIAN"; then
 984         # dmsetup installs 55-dm and 60-persistent-storage-dm on Debian/Ubuntu
 985         # and since buster/bionic 95-dm-notify.rules
 986         # see https://gitlab.com/debian-lvm/lvm2/blob/master/debian/patches/udev.patch
 987         inst_rules 55-dm.rules 60-persistent-storage-dm.rules 95-dm-notify.rules
 988     else
 989         inst_rules 10-dm.rules 13-dm-disk.rules 95-dm-notify.rules
 990     fi
 991     if get_bool "$LOOKS_LIKE_SUSE"; then
 992         inst_rules 60-persistent-storage.rules 61-persistent-storage-compat.rules 99-systemd.rules
 993     fi
 994 }
 995
 996 install_multipath() {
 997     instmods "=md" multipath
 998     image_install kpartx /lib/udev/kpartx_id lsmod mpathpersist multipath multipathd partx
 999     image_install "${ROOTLIBDIR:?}"/system/multipathd.{service,socket}
1000     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1001         inst_rules 56-dm-parts.rules 56-dm-mpath.rules 60-multipath.rules 68-del-part-nodes.rules 95-kpartx.rules
1002     else
1003         inst_rules 11-dm-mpath.rules 11-dm-parts.rules 62-multipath.rules 66-kpartx.rules 68-del-part-nodes.rules
1004     fi
1005     mkdir -p "${initdir:?}/etc/multipath"
1006
1007     local file
1008     while read -r file; do
1009         # Install libraries required by the given library
1010         inst_libs "$file"
1011         # Install the library itself and create necessary symlinks
1012         inst_library "$file"
1013     done < <(find /lib*/multipath -type f)
1014 }
1015
1016 install_lvm() {
1017     image_install lvm
1018     image_install "${ROOTLIBDIR:?}"/system/lvm2-lvmpolld.{service,socket}
1019     image_install "${ROOTLIBDIR:?}"/system/{blk-availability,lvm2-monitor}.service
1020     image_install -o "/lib/tmpfiles.d/lvm2.conf"
1021     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1022         inst_rules 56-lvm.rules 69-lvm-metad.rules
1023     else
1024         # Support the new udev autoactivation introduced in lvm 2.03.14
1025         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=67722b312390cdab29c076c912e14bd739c5c0f6
1026         # Static autoactivation (via lvm2-activation-generator) was dropped
1027         # in lvm 2.03.15
1028         # https://sourceware.org/git/?p=lvm2.git;a=commit;h=ee8fb0310c53ed003a43b324c99cdfd891dd1a7c
1029         if [[ -f /lib/udev/rules.d/69-dm-lvm.rules ]]; then
1030             inst_rules 11-dm-lvm.rules 69-dm-lvm.rules
1031         else
1032             image_install "${ROOTLIBDIR:?}"/system-generators/lvm2-activation-generator
1033             image_install "${ROOTLIBDIR:?}"/system/lvm2-pvscan@.service
1034             inst_rules 11-dm-lvm.rules 69-dm-lvm-metad.rules
1035         fi
1036     fi
1037     mkdir -p "${initdir:?}/etc/lvm"
1038 }
1039
1040 install_btrfs() {
1041     instmods btrfs
1042     # Not all utilities provided by btrfs-progs are listed here; extend the list
1043     # if necessary
1044     image_install btrfs btrfstune mkfs.btrfs
1045     inst_rules 64-btrfs-dm.rules
1046 }
1047
1048 install_iscsi() {
1049     # Install both client and server side stuff by default
1050     local inst="${1:-}"
1051     local file
1052
1053     # Install client-side stuff ("initiator" in iSCSI jargon) - Open-iSCSI in this case
1054     # (open-iscsi on Debian, iscsi-initiator-utils on Fedora, etc.)
1055     if [[ -z "$inst" || "$inst" =~ (client|initiator) ]]; then
1056         image_install iscsi-iname iscsiadm iscsid iscsistart
1057         image_install -o "${ROOTLIBDIR:?}"/system/iscsi-{init,onboot,shutdown}.service
1058         image_install "${ROOTLIBDIR:?}"/system/iscsid.{service,socket}
1059         image_install "${ROOTLIBDIR:?}"/system/iscsi.service
1060         mkdir -p "${initdir:?}"/var/lib/iscsi/{ifaces,isns,nodes,send_targets,slp,static}
1061         mkdir -p "${initdir:?}/etc/iscsi"
1062         echo "iscsid.startup = /bin/systemctl start iscsid.socket" >"${initdir:?}/etc/iscsi/iscsid.conf"
1063         # Since open-iscsi 2.1.2 [0] the initiator name should be generated via
1064         # a one-time service instead of distro package's post-install scripts.
1065         # However, some distros still use this approach even after this patch,
1066         # so prefer the already existing initiatorname.iscsi file if it exists.
1067         #
1068         # [0] https://github.com/open-iscsi/open-iscsi/commit/f37d5b653f9f251845db3f29b1a3dcb90ec89731
1069         if [[ ! -e /etc/iscsi/initiatorname.iscsi ]]; then
1070             image_install "${ROOTLIBDIR:?}"/system/iscsi-init.service
1071             if get_bool "$IS_BUILT_WITH_ASAN"; then
1072                 # The iscsi-init.service calls `sh` which might, in certain circumstances,
1073                 # pull in instrumented systemd NSS modules causing `sh` to fail. Let's mitigate
1074                 # this by pulling in an env file crafted by `create_asan_wrapper()` that
1075                 # (among others) pre-loads ASan's DSO.
1076                 mkdir -p "${initdir:?}/etc/systemd/system/iscsi-init.service.d/"
1077                 printf "[Service]\nEnvironmentFile=/usr/lib/systemd/systemd-asan-env" >"${initdir:?}/etc/systemd/system/iscsi-init.service.d/asan-env.conf"
1078             fi
1079         else
1080             inst_simple "/etc/iscsi/initiatorname.iscsi"
1081         fi
1082     fi
1083
1084     # Install server-side stuff ("target" in iSCSI jargon) - TGT in this case
1085     # (tgt on Debian, scsi-target-utils on Fedora, etc.)
1086     if [[ -z "$inst" || "$inst" =~ (server|target) ]]; then
1087         image_install tgt-admin tgt-setup-lun tgtadm tgtd tgtimg
1088         image_install -o /etc/sysconfig/tgtd
1089         image_install "${ROOTLIBDIR:?}"/system/tgtd.service
1090         mkdir -p "${initdir:?}/etc/tgt"
1091         touch "${initdir:?}"/etc/tgt/{tgtd,targets}.conf
1092         # Install perl modules required by tgt-admin
1093         #
1094         # Forgive me father for I have sinned. The monstrosity below appends
1095         # a perl snippet to the `tgt-admin` perl script on the fly, which
1096         # dumps a list of files (perl modules) required by `tgt-admin` at
1097         # the runtime plus any DSOs loaded via DynaLoader. This list is then
1098         # passed to `inst_simple` which installs the necessary files into the image
1099         #
1100         # shellcheck disable=SC2016
1101         while read -r file; do
1102             inst_simple "$file"
1103         done < <(perl -- <(cat "$(command -v tgt-admin)" <(echo -e 'use DynaLoader; print map { "$_\n" } values %INC; print join("\n", @DynaLoader::dl_shared_objects)')) -p | awk '/^\// { print $1 }')
1104     fi
1105 }
1106
1107 install_mdadm() {
1108     local unit
1109     local mdadm_units=(
1110         system/mdadm-grow-continue@.service
1111         system/mdadm-last-resort@.service
1112         system/mdadm-last-resort@.timer
1113         system/mdmon@.service
1114         system/mdmonitor-oneshot.service
1115         system/mdmonitor-oneshot.timer
1116         system/mdmonitor.service
1117         system-shutdown/mdadm.shutdown
1118     )
1119
1120     image_install mdadm mdmon
1121     inst_rules 01-md-raid-creating.rules 63-md-raid-arrays.rules 64-md-raid-assembly.rules 69-md-clustered-confirm-device.rules
1122     # Fedora/CentOS/RHEL ships this rule file
1123     [[ -f /lib/udev/rules.d/65-md-incremental.rules ]] && inst_rules 65-md-incremental.rules
1124
1125     for unit in "${mdadm_units[@]}"; do
1126         image_install "${ROOTLIBDIR:?}/$unit"
1127     done
1128 }
1129
1130 install_compiled_systemd() {
1131     dinfo "Install compiled systemd"
1132
1133     local ninja_bin
1134     ninja_bin="$(type -P ninja || type -P ninja-build)"
1135     if [[ -z "$ninja_bin" ]]; then
1136         dfatal "ninja was not found"
1137         exit 1
1138     fi
1139     (set -x; DESTDIR="$initdir" "$ninja_bin" -C "$BUILD_DIR" install)
1140
1141     # If we are doing coverage runs, copy over the binary notes files, as lcov expects to
1142     # find them in the same directory as the runtime data counts
1143     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1144         mkdir -p "${initdir}/${BUILD_DIR:?}/"
1145         rsync -am --include='*/' --include='*.gcno' --exclude='*' "${BUILD_DIR:?}/" "${initdir}/${BUILD_DIR:?}/"
1146         # Set effective & default ACLs for the build dir so unprivileged
1147         # processes can write gcda files with coverage stats
1148         setfacl -R -m 'd:o:rwX' -m 'o:rwX' "${initdir}/${BUILD_DIR:?}/"
1149     fi
1150 }
1151
1152 install_debian_systemd() {
1153     dinfo "Install debian systemd"
1154
1155     local files
1156
1157     while read -r deb; do
1158         files="$(dpkg-query -L "$deb" 2>/dev/null)" || continue
1159         ddebug "Install debian files from package $deb"
1160         for file in $files; do
1161             [ -e "$file" ] || continue
1162             [ ! -L "$file" ] && [ -d "$file" ] && continue
1163             inst "$file"
1164         done
1165     done < <(grep -E '^Package:' "${SOURCE_DIR}/debian/control" | cut -d ':' -f 2)
1166 }
1167
1168 install_suse_systemd() {
1169     local pkgs
1170
1171     dinfo "Install SUSE systemd"
1172
1173     pkgs=(
1174         systemd
1175         systemd-container
1176         systemd-coredump
1177         systemd-experimental
1178         systemd-journal-remote
1179         # Since commit fb6f25d7b979134a, systemd-resolved, which is shipped by
1180         # systemd-network sub-package on openSUSE, has its own testsuite.
1181         systemd-network
1182         systemd-portable
1183         udev
1184     )
1185
1186     for p in "${pkgs[@]}"; do
1187         rpm -q "$p" &>/dev/null || continue
1188
1189         ddebug "Install files from package $p"
1190         while read -r f; do
1191             [ -e "$f" ] || continue
1192             [ ! -L "$f" ] && [ -d "$f" ] && continue
1193             inst "$f"
1194         done < <(rpm -ql "$p")
1195     done
1196
1197     dinfo "Install the data needed by the tests at runtime"
1198     inst_recursive "${SOURCE_DIR}/testdata"
1199
1200     # On openSUSE, this directory is not created at package install, at least
1201     # for now.
1202     mkdir -p "$initdir/var/log/journal/remote"
1203 }
1204
1205 install_distro_systemd() {
1206     dinfo "Install distro systemd"
1207
1208     if get_bool "$LOOKS_LIKE_DEBIAN"; then
1209         install_debian_systemd
1210     elif get_bool "$LOOKS_LIKE_SUSE"; then
1211         install_suse_systemd
1212     else
1213         dfatal "NO_BUILD not supported for this distro"
1214         exit 1
1215     fi
1216 }
1217
1218 install_systemd() {
1219     dinfo "Install systemd"
1220     if get_bool "$NO_BUILD"; then
1221         install_distro_systemd
1222     else
1223         install_compiled_systemd
1224     fi
1225
1226     # Remove unneeded documentation
1227     rm -fr "${initdir:?}"/usr/share/{man,doc}
1228
1229     # Enable debug logging in PID1
1230     mkdir -p "$initdir/etc/systemd/system.conf.d/"
1231     echo -ne "[Manager]\nLogLevel=debug\n" >"$initdir/etc/systemd/system.conf.d/10-log-level.conf"
1232     if [[ -n "$TEST_SYSTEMD_LOG_LEVEL" ]]; then
1233         echo DefaultEnvironment=SYSTEMD_LOG_LEVEL="$TEST_SYSTEMD_LOG_LEVEL" >>"$initdir/etc/systemd/system.conf.d/99-log-level.conf"
1234     fi
1235     # Store coredumps in journal
1236     mkdir -p "$initdir/etc/systemd/coredump.conf.d/"
1237     echo -ne "[Coredump]\nStorage=journal\n" >"$initdir/etc/systemd/coredump.conf.d/10-storage-journal.conf"
1238     # Propagate SYSTEMD_UNIT_PATH to user systemd managers
1239     mkdir -p "$initdir/etc/systemd/system/user@.service.d/"
1240     echo -ne "[Service]\nPassEnvironment=SYSTEMD_UNIT_PATH\n" >"$initdir/etc/systemd/system/user@.service.d/99-SYSTEMD_UNIT_PATH.conf"
1241
1242     # When built with gcov, disable ProtectSystem= and ProtectHome= in the test
1243     # images, since it prevents gcov to write the coverage reports (*.gcda
1244     # files)
1245     if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1246         mkdir -p "$initdir/etc/systemd/system/service.d/"
1247         echo -ne "[Service]\nProtectSystem=no\nProtectHome=no\n" >"$initdir/etc/systemd/system/service.d/99-gcov-override.conf"
1248         # Similarly, set ReadWritePaths= to the $BUILD_DIR in the test image
1249         # to make the coverage work with units using DynamicUser=yes. Do this
1250         # only for services with test- prefix, as setting this system-wide
1251         # has many undesirable side-effects, as it creates its own namespace.
1252         mkdir -p "$initdir/etc/systemd/system/test-.service.d/"
1253         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/system/test-.service.d/99-gcov-rwpaths-override.conf"
1254         # Ditto, but for the user daemon
1255         mkdir -p "$initdir/etc/systemd/user/test-.service.d/"
1256         echo -ne "[Service]\nReadWritePaths=${BUILD_DIR:?}\n" >"$initdir/etc/systemd/user/test-.service.d/99-gcov-rwpaths-override.conf"
1257         # Bind the $BUILD_DIR into nspawn containers that are executed using
1258         # machinectl. Unfortunately, the .nspawn files don't support drop-ins
1259         # so we have to inject the bind mount directly into
1260         # the systemd-nspawn@.service unit.
1261         cp "$initdir/usr/lib/systemd/system/systemd-nspawn@.service" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1262         sed -ri "s/^ExecStart=.+$/& --bind=${BUILD_DIR//\//\\\/}/" "$initdir/etc/systemd/system/systemd-nspawn@.service"
1263         # Pass the $BUILD_DIR as $COVERAGE_BUILD_DIR env variable to the system
1264         # manager, similarly to what we do with $ASAN_RT_PATH during sanitized
1265         # builds
1266         mkdir -p "$initdir/etc/systemd/system.conf.d/"
1267         echo -ne "[Manager]\nDefaultEnvironment=COVERAGE_BUILD_DIR=$BUILD_DIR\n" >"$initdir/etc/systemd/system.conf.d/99-COVERAGE_BUILD_DIR.conf"
1268     fi
1269
1270     # If we're built with -Dportabled=false, tests with systemd-analyze
1271     # --profile will fail. Since we need just the profile (text) files, let's
1272     # copy them into the image if they don't exist there.
1273     local portable_dir="${initdir:?}${ROOTLIBDIR:?}/portable"
1274     if [[ ! -d "$portable_dir/profile/strict" ]]; then
1275         dinfo "Couldn't find portable profiles in the test image"
1276         dinfo "Copying them directly from the source tree"
1277         mkdir -p "$portable_dir"
1278         cp -frv "${SOURCE_DIR:?}/src/portable/profile" "$portable_dir"
1279     fi
1280 }
1281
1282 get_ldpath() {
1283     local rpath
1284     rpath="$(objdump -p "${1:?}" 2>/dev/null | awk "/R(UN)?PATH/ { print \"$initdir\" \$2 }" | paste -sd :)"
1285
1286     if [ -z "$rpath" ] ; then
1287         echo "$BUILD_DIR"
1288     else
1289         echo "$rpath"
1290     fi
1291 }
1292
1293 install_missing_libraries() {
1294     dinfo "Install missing libraries"
1295     # install possible missing libraries
1296     for i in "${initdir:?}"{,/usr}/{sbin,bin}/* "$initdir"{,/usr}/lib/systemd/{,tests/unit-tests/{,manual/,unsafe/}}*; do
1297         LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$(get_ldpath "$i")" inst_libs "$i"
1298     done
1299
1300     # Install libgcc_s.so if available, since it's dlopen()ed by libpthread
1301     # and might cause unexpected failures during pthread_exit()/pthread_cancel()
1302     # if not present
1303     # See: https://github.com/systemd/systemd/pull/23858
1304     while read -r libgcc_s; do
1305         [[ -e "$libgcc_s" ]] && inst_library "$libgcc_s"
1306     done < <(ldconfig -p | awk '/\/libgcc_s.so.1$/ { print $4 }')
1307
1308     local lib path
1309     # A number of dependencies is now optional via dlopen, so the install
1310     # script will not pick them up, since it looks at linkage.
1311     for lib in libcryptsetup libidn libidn2 pwquality libqrencode tss2-esys tss2-rc tss2-mu tss2-tcti-device libfido2 libbpf libelf libdw xkbcommon p11-kit-1; do
1312         ddebug "Searching for $lib via pkg-config"
1313         if pkg-config --exists "$lib"; then
1314                 path="$(pkg-config --variable=libdir "$lib")"
1315                 if [ -z "${path}" ]; then
1316                     ddebug "$lib.pc does not contain a libdir variable, skipping"
1317                     continue
1318                 fi
1319
1320                 if ! [[ ${lib} =~ ^lib ]]; then
1321                         lib="lib${lib}"
1322                 fi
1323                 # p11-kit-1's .so doesn't have the API level in the name
1324                 if [[ ${lib} =~ p11-kit-1$ ]]; then
1325                         lib="libp11-kit"
1326                 fi
1327                 # Some pkg-config files are broken and give out the wrong paths
1328                 # (eg: libcryptsetup), so just ignore them
1329                 inst_libs "${path}/${lib}.so" || true
1330                 inst_library "${path}/${lib}.so" || true
1331
1332                 if [[ "$lib" == "libxkbcommon" ]]; then
1333                     install_x11_keymaps full
1334                 fi
1335         else
1336             ddebug "$lib.pc not found, skipping"
1337             continue
1338         fi
1339     done
1340
1341     # Install extra openssl 3 stuff
1342     path="$(pkg-config --variable=libdir libcrypto)"
1343     inst_simple "${path}/ossl-modules/legacy.so" || true
1344     inst_simple "${path}/ossl-modules/fips.so" || true
1345     inst_simple "${path}/engines-3/afalg.so" || true
1346     inst_simple "${path}/engines-3/capi.so" || true
1347     inst_simple "${path}/engines-3/loader_attic.so" || true
1348     inst_simple "${path}/engines-3/padlock.so" || true
1349
1350     # Binaries from mtools depend on the gconv modules to translate between codepages. Because there's no
1351     # pkg-config file for these, we copy every gconv/ directory we can find in /usr/lib and /usr/lib64.
1352     # shellcheck disable=SC2046
1353     inst_recursive $(find /usr/lib* -name gconv 2>/dev/null)
1354 }
1355
1356 cleanup_loopdev() {
1357     if [ -n "${LOOPDEV:=}" ]; then
1358         ddebug "losetup -d $LOOPDEV"
1359         losetup -d "${LOOPDEV}"
1360         unset LOOPDEV
1361     fi
1362 }
1363
1364 add_at_exit_handler cleanup_loopdev
1365
1366 create_empty_image() {
1367     if [ -z "${IMAGE_NAME:=}" ]; then
1368         echo "create_empty_image: \$IMAGE_NAME not set"
1369         exit 1
1370     fi
1371
1372     # Partition sizes are in MiBs
1373     local root_size=768
1374     local data_size=100
1375     if ! get_bool "$NO_BUILD"; then
1376         if meson configure "${BUILD_DIR:?}" | grep 'static-lib\|standalone-binaries' | awk '{ print $2 }' | grep -q 'true'; then
1377             root_size=$((root_size+=200))
1378         fi
1379         if meson configure "${BUILD_DIR:?}" | grep 'link-.*-shared' | awk '{ print $2 }' | grep -q 'false'; then
1380             root_size=$((root_size+=200))
1381         fi
1382         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
1383             root_size=$((root_size+=250))
1384         fi
1385         if get_bool "$IS_BUILT_WITH_ASAN"; then
1386             root_size=$((root_size * 2))
1387         fi
1388     fi
1389
1390     if [ "$IMAGE_NAME" = "repart" ]; then
1391         root_size=$((root_size+=1000))
1392     fi
1393
1394     echo "Setting up ${IMAGE_PUBLIC:?} (${root_size} MB)"
1395     rm -f "${IMAGE_PRIVATE:?}" "$IMAGE_PUBLIC"
1396
1397     # Create the blank file to use as a root filesystem
1398     truncate -s "${root_size}M" "$IMAGE_PUBLIC"
1399
1400     LOOPDEV=$(losetup --show -P -f "$IMAGE_PUBLIC")
1401     [ -b "$LOOPDEV" ] || return 1
1402     # Create two partitions - a root one and a data one (utilized by some tests)
1403     sfdisk "$LOOPDEV" <<EOF
1404 label: gpt
1405 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=root size=$((root_size - data_size))M bootable
1406 type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 name=data
1407 EOF
1408
1409     udevadm settle
1410
1411     local label=(-L systemd_boot)
1412     # mkfs.reiserfs doesn't know -L. so, use --label instead
1413     [[ "$FSTYPE" == "reiserfs" ]] && label=(--label systemd_boot)
1414     if ! mkfs -t "${FSTYPE}" "${label[@]}" "${LOOPDEV}p1" -q; then
1415         dfatal "Failed to mkfs -t ${FSTYPE}"
1416         exit 1
1417     fi
1418 }
1419
1420 mount_initdir() {
1421     if [ -z "${LOOPDEV:=}" ]; then
1422         [ -e "${IMAGE_PRIVATE:?}" ] && image="$IMAGE_PRIVATE" || image="${IMAGE_PUBLIC:?}"
1423         LOOPDEV="$(losetup --show -P -f "$image")"
1424         [ -b "$LOOPDEV" ] || return 1
1425
1426         udevadm settle
1427     fi
1428
1429     if ! mountpoint -q "${initdir:?}"; then
1430         mkdir -p "$initdir"
1431         mount "${LOOPDEV}p1" "$initdir"
1432         TEST_SETUP_CLEANUP_ROOTDIR=1
1433     fi
1434 }
1435
1436 cleanup_initdir() {
1437     # only umount if create_empty_image_rootdir() was called to mount it
1438     if get_bool "$TEST_SETUP_CLEANUP_ROOTDIR"; then
1439         _umount_dir "${initdir:?}"
1440     fi
1441 }
1442
1443 umount_loopback() {
1444     # unmount the loopback device from all places. Otherwise we risk file
1445     # system corruption.
1446     for device in $(losetup -l | awk '$6=="'"${IMAGE_PUBLIC:?}"'" {print $1}'); do
1447         ddebug "Unmounting all uses of $device"
1448         mount | awk '/^'"${device}"'p/{print $1}' | xargs --no-run-if-empty umount -v
1449     done
1450 }
1451
1452 create_empty_image_rootdir() {
1453     create_empty_image
1454     mount_initdir
1455 }
1456
1457 check_asan_reports() {
1458     local ret=0
1459     local root="${1:?}"
1460
1461     if get_bool "$IS_BUILT_WITH_ASAN"; then
1462         ls -l "$root"
1463         if [[ -e "$root/systemd.asan.log.1" ]]; then
1464             cat "$root/systemd.asan.log.1"
1465             ret=$((ret+1))
1466         fi
1467
1468         journald_report="$(find "$root" -name "systemd-journald.*san.log*" -exec cat {} \;)"
1469         if [[ -n "$journald_report" ]]; then
1470             printf "%s\n" "$journald_report"
1471             cat "$root/systemd-journald.out" || :
1472             ret=$((ret+1))
1473         fi
1474
1475         # May 08 13:23:31 H testleak[2907148]: SUMMARY: AddressSanitizer: 4 byte(s) leaked in 1 allocation(s).
1476         pids="$(
1477             "$JOURNALCTL" -D "$root/var/log/journal" --grep 'SUMMARY: .*Sanitizer:' |
1478                    grep -v -E 'dbus-daemon|dbus-broker-launch' |
1479                    sed -r -n 's/.* .+\[([0-9]+)\]: SUMMARY:.*/\1/p'
1480         )"
1481
1482         if [[ -n "$pids" ]]; then
1483             ret=$((ret+1))
1484             for pid in $pids; do
1485                 "$JOURNALCTL" -D "$root/var/log/journal" _PID="$pid" --no-pager
1486             done
1487         fi
1488     fi
1489
1490     return $ret
1491 }
1492
1493 check_coverage_reports() {
1494     local root="${1:?}"
1495
1496     if get_bool "$NO_BUILD"; then
1497         return 0
1498     fi
1499     if ! get_bool "$IS_BUILT_WITH_COVERAGE"; then
1500         return 0
1501     fi
1502
1503     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1504         dest="${ARTIFACT_DIRECTORY}/${testname:?}.coverage-info"
1505     else
1506         dest="${TESTDIR:?}/coverage-info"
1507     fi
1508
1509     if [[ ! -e "${TESTDIR:?}/coverage-base" ]]; then
1510         # This shouldn't happen, as the report is generated during the setup
1511         # phase (test_setup()).
1512         derror "Missing base coverage report"
1513         return 1
1514     fi
1515
1516     # Create a coverage report that will later be uploaded. Remove info about
1517     # system libraries/headers, as we don't really care about them.
1518     lcov --directory "${root}/${BUILD_DIR:?}" --capture --output-file "${dest}.new"
1519     if [[ -f "$dest" ]]; then
1520         # If the destination report file already exists, don't overwrite it, but
1521         # merge it with the already present one - this usually happens when
1522         # running both "parts" of a test in one run (the qemu and the nspawn part).
1523         lcov --add-tracefile "${dest}" --add-tracefile "${dest}.new" -o "${dest}"
1524     else
1525         # If there's no prior coverage report, merge the new one with the base
1526         # report we did during the setup phase (see test_setup()).
1527         lcov --add-tracefile "${TESTDIR:?}/coverage-base" --add-tracefile "${dest}.new" -o "${dest}"
1528     fi
1529     lcov --remove "$dest" -o "$dest" '/usr/include/*' '/usr/lib/*'
1530     rm -f "${dest}.new"
1531
1532     # If the test logs contain lines like:
1533     #
1534     # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open
1535     #
1536     # it means we're possibly missing some coverage since gcov can't write the stats,
1537     # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes,
1538     # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see
1539     # `setfacl` stuff in install_compiled_systemd().
1540     #
1541     # Also, a note: some tests, like TEST-46, overmount /home with tmpfs, which
1542     # means if your build dir is under /home/your-user (which is usually the
1543     # case) you might get bogus errors and missing coverage.
1544     if ! get_bool "${IGNORE_MISSING_COVERAGE:=}" && \
1545        "${JOURNALCTL:?}" -q --no-pager -D "${root:?}/var/log/journal" --grep "profiling:.+?gcda:[Cc]annot open"; then
1546         derror "Detected possibly missing coverage, check the journal"
1547         return 1
1548     fi
1549
1550     return 0
1551 }
1552
1553 save_journal() {
1554     # Default to always saving journal
1555     local save="yes"
1556
1557     if [ "${TEST_SAVE_JOURNAL}" = "no" ]; then
1558         save="no"
1559     elif [ "${TEST_SAVE_JOURNAL}" = "fail" ] && [ "$2" = "0" ]; then
1560         save="no"
1561     fi
1562
1563     if [ -n "${ARTIFACT_DIRECTORY}" ]; then
1564         dest="${ARTIFACT_DIRECTORY}/${testname:?}.journal"
1565     else
1566         dest="${TESTDIR:?}/system.journal"
1567     fi
1568
1569     for j in "${1:?}"/*; do
1570         if get_bool "$save"; then
1571             if [ "$SYSTEMD_JOURNAL_REMOTE" = "" ]; then
1572                 cp -a "$j" "$dest"
1573             else
1574                 "$SYSTEMD_JOURNAL_REMOTE" -o "$dest" --getter="$JOURNALCTL -o export -D $j"
1575             fi
1576         fi
1577
1578         if [ -n "${TEST_SHOW_JOURNAL}" ]; then
1579             echo "---- $j ----"
1580             "$JOURNALCTL" --no-pager -o short-monotonic --no-hostname --priority="${TEST_SHOW_JOURNAL}" -D "$j"
1581         fi
1582
1583         rm -r "$j"
1584     done
1585
1586     if ! get_bool "$save"; then
1587         return 0
1588     fi
1589
1590     if [ -n "${SUDO_USER}" ]; then
1591         setfacl -m "user:${SUDO_USER:?}:r-X" "$dest"*
1592     fi
1593
1594     # we want to print this sometime later, so save this in a variable
1595     JOURNAL_LIST="$(ls -l "$dest"*)"
1596 }
1597
1598 check_result_common() {
1599     local workspace="${1:?}"
1600     local ret
1601
1602     if [ -s "$workspace/failed" ]; then
1603         # Non-empty …/failed has highest priority
1604         cp -a "$workspace/failed" "${TESTDIR:?}/"
1605         if [ -n "${SUDO_USER}" ]; then
1606             setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}/"failed
1607         fi
1608         ret=1
1609     elif get_bool "$TIMED_OUT"; then
1610         echo "(timeout)" >"${TESTDIR:?}/failed"
1611         ret=2
1612     elif [ -e "$workspace/testok" ]; then
1613         # …/testok always counts (but with lower priority than …/failed)
1614         ret=0
1615     elif [ -e "$workspace/skipped" ]; then
1616         # …/skipped always counts (a message is expected)
1617         echo "${TESTNAME:?} was skipped:"
1618         cat "$workspace/skipped"
1619         ret=0
1620     else
1621         echo "(failed; see logs)" >"${TESTDIR:?}/failed"
1622         ret=3
1623     fi
1624
1625     check_asan_reports "$workspace" || ret=4
1626
1627     check_coverage_reports "$workspace" || ret=5
1628
1629     save_journal "$workspace/var/log/journal" $ret
1630
1631     if [ -d "${ARTIFACT_DIRECTORY}" ] && [ -f "$workspace/strace.out" ]; then
1632         cp "$workspace/strace.out" "${ARTIFACT_DIRECTORY}/"
1633     fi
1634
1635     if [ ${ret:?} != 0 ] && [ -f "$TESTDIR/failed" ]; then
1636         echo -n "${TESTNAME:?}: "
1637         cat "$TESTDIR/failed"
1638     fi
1639     echo "${JOURNAL_LIST:-"No journals were saved"}"
1640
1641     return ${ret:?}
1642 }
1643
1644 check_result_nspawn() {
1645     local workspace="${1:?}"
1646     local ret
1647
1648     # Run a test-specific checks if defined by check_result_nspawn_hook()
1649     if declare -F check_result_nspawn_hook >/dev/null; then
1650         if ! check_result_nspawn_hook "${workspace}"; then
1651             derror "check_result_nspawn_hook() returned with EC > 0"
1652             ret=4
1653         fi
1654     fi
1655
1656     check_result_common "${workspace}"
1657     ret=$?
1658
1659     _umount_dir "${initdir:?}"
1660
1661     return $ret
1662 }
1663
1664 # can be overridden in specific test
1665 check_result_qemu() {
1666     local ret
1667     mount_initdir
1668
1669     # Run a test-specific checks if defined by check_result_qemu_hook()
1670     if declare -F check_result_qemu_hook >/dev/null; then
1671         if ! check_result_qemu_hook "${initdir:?}"; then
1672             derror "check_result_qemu_hook() returned with EC > 0"
1673             ret=4
1674         fi
1675     fi
1676
1677     check_result_common "${initdir:?}"
1678     ret=$?
1679
1680     _umount_dir "${initdir:?}"
1681
1682     return $ret
1683 }
1684
1685 check_result_nspawn_unittests() {
1686     local workspace="${1:?}"
1687     local ret=1
1688
1689     [[ -e "$workspace/testok" ]] && ret=0
1690
1691     if [[ -s "$workspace/failed" ]]; then
1692         ret=$((ret + 1))
1693         echo "=== Failed test log ==="
1694         cat "$workspace/failed"
1695     else
1696         if [[ -s "$workspace/skipped" ]]; then
1697             echo "=== Skipped test log =="
1698             cat "$workspace/skipped"
1699             # We might have only skipped tests - that should not fail the job
1700             ret=0
1701         fi
1702         if [[ -s "$workspace/testok" ]]; then
1703             echo "=== Passed tests ==="
1704             cat "$workspace/testok"
1705         fi
1706     fi
1707
1708     get_bool "${TIMED_OUT:=}" && ret=1
1709     check_coverage_reports "$workspace" || ret=5
1710
1711     save_journal "$workspace/var/log/journal" $ret
1712
1713     _umount_dir "${initdir:?}"
1714
1715     return $ret
1716 }
1717
1718 check_result_qemu_unittests() {
1719     local ret=1
1720
1721     mount_initdir
1722     [[ -e "${initdir:?}/testok" ]] && ret=0
1723
1724     if [[ -s "$initdir/failed" ]]; then
1725         ret=$((ret + 1))
1726         echo "=== Failed test log ==="
1727         cat "$initdir/failed"
1728     else
1729         if [[ -s "$initdir/skipped" ]]; then
1730             echo "=== Skipped test log =="
1731             cat "$initdir/skipped"
1732             # We might have only skipped tests - that should not fail the job
1733             ret=0
1734         fi
1735         if [[ -s "$initdir/testok" ]]; then
1736             echo "=== Passed tests ==="
1737             cat "$initdir/testok"
1738         fi
1739     fi
1740
1741     get_bool "${TIMED_OUT:=}" && ret=1
1742     check_coverage_reports "$initdir" || ret=5
1743
1744     save_journal "$initdir/var/log/journal" $ret
1745
1746     _umount_dir "$initdir"
1747
1748     return $ret
1749 }
1750
1751 create_rc_local() {
1752     dinfo "Create rc.local"
1753     mkdir -p "${initdir:?}/etc/rc.d"
1754     cat >"$initdir/etc/rc.d/rc.local" <<EOF
1755 #!/usr/bin/env bash
1756 exit 0
1757 EOF
1758     chmod 0755 "$initdir/etc/rc.d/rc.local"
1759 }
1760
1761 install_execs() {
1762     ddebug "Install executables from the service files"
1763
1764     local pkg_config_path="${BUILD_DIR:?}/src/core/"
1765     local systemunitdir userunitdir exe
1766     systemunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemdsystemunitdir systemd)"
1767     userunitdir="$(PKG_CONFIG_PATH="$pkg_config_path" pkg-config --variable=systemduserunitdir systemd)"
1768     while read -r exe; do
1769         # some {rc,halt}.local scripts and programs are okay to not exist, the rest should
1770         # also, plymouth is pulled in by rescue.service, but even there the exit code
1771         # is ignored; as it's not present on some distros, don't fail if it doesn't exist
1772         dinfo "Attempting to install $exe (based on unit file reference)"
1773         inst "$exe" || [ "${exe%.local}" != "$exe" ] || [ "${exe%systemd-update-done}" != "$exe" ] || [ "${exe##*/}" == "plymouth" ]
1774     done < <(sed -r -n 's|^Exec[a-zA-Z]*=[@+!-]*([^ ]+).*|\1|gp' "${initdir:?}"/{"$systemunitdir","$userunitdir"}/*.service | sort -u)
1775 }
1776
1777 generate_module_dependencies() {
1778     dinfo "Generate modules dependencies"
1779     if [[ -d "${initdir:?}/lib/modules/${KERNEL_VER:?}" ]] && \
1780         ! depmod -a -b "$initdir" "$KERNEL_VER"; then
1781             dfatal "\"depmod -a $KERNEL_VER\" failed."
1782             exit 1
1783     fi
1784 }
1785
1786 install_depmod_files() {
1787     dinfo "Install depmod files"
1788     inst "/lib/modules/${KERNEL_VER:?}/modules.order"
1789     inst "/lib/modules/$KERNEL_VER/modules.builtin"
1790 }
1791
1792 install_plymouth() {
1793     dinfo "Install plymouth"
1794     # install plymouth, if found... else remove plymouth service files
1795     # if [ -x /usr/libexec/plymouth/plymouth-populate-initrd ]; then
1796     #     PLYMOUTH_POPULATE_SOURCE_FUNCTIONS="$TEST_BASE_DIR/test-functions" \
1797     #         /usr/libexec/plymouth/plymouth-populate-initrd -t $initdir
1798     #         image_install plymouth plymouthd
1799     # else
1800         rm -f "${initdir:?}"/{usr/lib,lib,etc}/systemd/system/plymouth* "$initdir"/{usr/lib,lib,etc}/systemd/system/*/plymouth*
1801     # fi
1802 }
1803
1804 install_haveged() {
1805     # If haveged is installed, it's probably included in initrd and needs to be
1806     # installed in the image too.
1807     if [ -x /usr/sbin/haveged ]; then
1808         dinfo "Install haveged files"
1809         inst /usr/sbin/haveged
1810         for u in /usr/lib/systemd/system/haveged*; do
1811             inst "$u"
1812         done
1813     fi
1814 }
1815
1816 install_ld_so_conf() {
1817     dinfo "Install /etc/ld.so.conf*"
1818     cp -a /etc/ld.so.conf* "${initdir:?}/etc"
1819     ldconfig -r "$initdir"
1820 }
1821
1822 install_testuser() {
1823     dinfo "Set up a test user"
1824     # create unprivileged user for user manager tests
1825     mkdir -p "${initdir:?}/etc/sysusers.d"
1826     cat >"$initdir/etc/sysusers.d/testuser.conf" <<EOF
1827 u testuser    4711     "Test User" /home/testuser
1828 EOF
1829
1830     mkdir -p "$initdir/home/testuser"
1831     chmod 0700 "$initdir/home/testuser"
1832     chown 4711:4711 "$initdir/home/testuser"
1833 }
1834
1835 install_config_files() {
1836     dinfo "Install config files"
1837     inst /etc/sysconfig/init || :
1838     inst /etc/passwd
1839     inst /etc/shadow
1840     inst_any /etc/login.defs /usr/etc/login.defs
1841     inst /etc/group
1842     inst /etc/shells
1843     inst_any /etc/nsswitch.conf /usr/etc/nsswitch.conf
1844     inst /etc/pam.conf || :
1845     inst_any /etc/os-release /usr/lib/os-release
1846     inst /etc/localtime
1847     # we want an empty environment
1848     : >"${initdir:?}/etc/environment"
1849     : >"$initdir/etc/machine-id"
1850     : >"$initdir/etc/resolv.conf"
1851
1852     # set the hostname
1853     echo 'H' >"$initdir/etc/hostname"
1854
1855     # let's set up just one image with the traditional verbose output
1856     if [ "${IMAGE_NAME:?}" != "basic" ]; then
1857         mkdir -p "$initdir/etc/systemd/system.conf.d"
1858         echo -e '[Manager]\nStatusUnitFormat=name' >"$initdir/etc/systemd/system.conf.d/status.conf"
1859     fi
1860 }
1861
1862 install_basic_tools() {
1863     dinfo "Install basic tools"
1864     image_install "${BASICTOOLS[@]}"
1865     image_install -o sushell
1866     # in Debian ldconfig is just a shell script wrapper around ldconfig.real
1867     image_install -o ldconfig.real
1868 }
1869
1870 install_debug_tools() {
1871     dinfo "Install debug tools"
1872     image_install -o "${DEBUGTOOLS[@]}"
1873
1874     if get_bool "$INTERACTIVE_DEBUG"; then
1875         # Set default TERM from vt220 to linux, so at least basic key shortcuts work
1876         local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
1877         mkdir -p "$getty_override"
1878         echo -e "[Service]\nEnvironment=TERM=linux" >"$getty_override/default-TERM.conf"
1879         echo 'export TERM=linux' >>"$initdir/etc/profile"
1880
1881         if command -v resize >/dev/null; then
1882             image_install resize
1883             echo "resize" >>"$initdir/etc/profile"
1884         fi
1885
1886         # Sometimes we might end up with plymouthd still running (especially
1887         # with the initrd -> asan_wrapper -> systemd transition), which will eat
1888         # our inputs and make debugging via tty impossible. Let's fix this by
1889         # killing plymouthd explicitly for the interactive sessions.
1890         # Note: we can't use pkill/pidof/etc. here due to a bug in libasan, see:
1891         #   - https://github.com/llvm/llvm-project/issues/49223
1892         #   - https://bugzilla.redhat.com/show_bug.cgi?id=2098125
1893         local plymouth_unit="${initdir:?}/etc/systemd/system/kill-plymouth.service"
1894         cat >"$plymouth_unit" <<EOF
1895 [Unit]
1896 After=multi-user.target
1897
1898 [Service]
1899 ExecStart=sh -c 'killall --verbose plymouthd || :'
1900
1901 [Install]
1902 WantedBy=multi-user.target
1903 EOF
1904         "${SYSTEMCTL:?}" enable --root "${initdir:?}" kill-plymouth.service
1905     fi
1906 }
1907
1908 install_libnss() {
1909     dinfo "Install libnss"
1910     # install libnss_files for login
1911     local NSS_LIBS
1912     mapfile -t NSS_LIBS < <(LD_DEBUG=files getent passwd 2>&1 >/dev/null | sed -n '/calling init: .*libnss_/ {s!^.* /!/!; p}')
1913     if [[ ${#NSS_LIBS[@]} -gt 0 ]]; then
1914         image_install "${NSS_LIBS[@]}"
1915     fi
1916 }
1917
1918 install_dbus() {
1919     dinfo "Install dbus"
1920     inst "${ROOTLIBDIR:?}/system/dbus.socket"
1921
1922     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1923     if [ -f "$ROOTLIBDIR/system/dbus-broker.service" ]; then
1924         inst "$ROOTLIBDIR/system/dbus-broker.service"
1925         inst_symlink /etc/systemd/system/dbus.service
1926         inst /usr/bin/dbus-broker
1927         inst /usr/bin/dbus-broker-launch
1928     elif [ -f "$ROOTLIBDIR/system/dbus-daemon.service" ]; then
1929         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1930         inst "$ROOTLIBDIR/system/dbus-daemon.service"
1931         # Alias symlink
1932         inst_symlink /etc/systemd/system/dbus.service
1933     else
1934         inst "$ROOTLIBDIR/system/dbus.service"
1935     fi
1936
1937     while read -r file; do
1938         inst "$file"
1939     done < <(find /etc/dbus-1 /usr/share/dbus-1 -xtype f 2>/dev/null)
1940
1941     # setup policy for Type=dbus test
1942     mkdir -p "${initdir:?}/etc/dbus-1/system.d"
1943     cat >"$initdir/etc/dbus-1/system.d/systemd.test.ExecStopPost.conf" <<EOF
1944 <?xml version="1.0"?>
1945 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
1946         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
1947 <busconfig>
1948     <policy user="root">
1949         <allow own="systemd.test.ExecStopPost"/>
1950     </policy>
1951 </busconfig>
1952 EOF
1953
1954     # If we run without KVM, bump the service start timeout
1955     if ! get_bool "$QEMU_KVM"; then
1956         cat >"$initdir/etc/dbus-1/system.d/service.timeout.conf" <<EOF
1957 <?xml version="1.0"?>
1958 <!DOCTYPE busconfig PUBLIC "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
1959         "https://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
1960 <busconfig>
1961     <limit name="service_start_timeout">120000</limit>
1962 </busconfig>
1963 EOF
1964         # Bump the client-side timeout in sd-bus as well
1965         mkdir -p "$initdir/etc/systemd/system.conf.d"
1966         echo -e '[Manager]\nDefaultEnvironment=SYSTEMD_BUS_TIMEOUT=120' >"$initdir/etc/systemd/system.conf.d/bus-timeout.conf"
1967     fi
1968 }
1969
1970 install_user_dbus() {
1971     dinfo "Install user dbus"
1972     local userunitdir
1973     if ! userunitdir="$(pkg-config --variable=systemduserunitdir systemd)"; then
1974         dwarn "WARNING! Cannot determine userunitdir from pkg-config, assuming /usr/lib/systemd/user"
1975         userunitdir=/usr/lib/systemd/user
1976     fi
1977
1978     inst "$userunitdir/dbus.socket"
1979     inst_symlink "$userunitdir/sockets.target.wants/dbus.socket" || inst_symlink /etc/systemd/user/sockets.target.wants/dbus.socket
1980
1981     # Append the After= dependency on dbus in case it isn't already set up
1982     mkdir -p "${initdir:?}/etc/systemd/system/user@.service.d/"
1983     cat >"$initdir/etc/systemd/system/user@.service.d/dbus.conf" <<EOF
1984 [Unit]
1985 After=dbus.service
1986 EOF
1987
1988     # Newer Fedora versions use dbus-broker by default. Let's install it if it's available.
1989     if [ -f "$userunitdir/dbus-broker.service" ]; then
1990         inst "$userunitdir/dbus-broker.service"
1991         inst_symlink /etc/systemd/user/dbus.service
1992     elif [ -f "${ROOTLIBDIR:?}/system/dbus-daemon.service" ]; then
1993         # Fedora rawhide replaced dbus.service with dbus-daemon.service
1994         inst "$userunitdir/dbus-daemon.service"
1995         # Alias symlink
1996         inst_symlink /etc/systemd/user/dbus.service
1997     else
1998         inst "$userunitdir/dbus.service"
1999     fi
2000 }
2001
2002 install_pam() {
2003     dinfo "Install PAM"
2004     local paths=()
2005
2006     if get_bool "$LOOKS_LIKE_DEBIAN" && type -p dpkg-architecture &>/dev/null; then
2007         paths+=("/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security")
2008     else
2009         paths+=(/lib*/security)
2010     fi
2011
2012     for d in /etc/pam.d /{usr/,}etc/security /usr/{etc,lib}/pam.d; do
2013         [ -d "$d" ] && paths+=("$d")
2014     done
2015
2016     while read -r file; do
2017         inst "$file"
2018     done < <(find "${paths[@]}" -xtype f)
2019
2020     # pam_unix depends on unix_chkpwd.
2021     # see http://www.linux-pam.org/Linux-PAM-html/sag-pam_unix.html
2022     image_install -o unix_chkpwd
2023
2024     # set empty root password for easy debugging
2025     sed -i 's/^root:x:/root::/' "${initdir:?}/etc/passwd"
2026
2027     # And make sure pam_unix will accept it by making sure that
2028     # the PAM module has the nullok option.
2029     for d in /etc/pam.d /usr/{etc,lib}/pam.d; do
2030         [ -d "$initdir/$d" ] || continue
2031         sed -i '/^auth.*pam_unix.so/s/$/ nullok/' "$initdir/$d"/*
2032     done
2033 }
2034
2035 install_locales() {
2036     # install only C.UTF-8 and English locales
2037     dinfo "Install locales"
2038
2039     if command -v meson >/dev/null \
2040             && (meson configure "${BUILD_DIR:?}" | grep 'localegen-path */') \
2041             || get_bool "$LOOKS_LIKE_DEBIAN"; then
2042         # locale-gen support
2043         image_install -o locale-gen localedef
2044         inst /etc/locale.gen || :
2045         inst /usr/share/i18n/SUPPORTED || :
2046         inst_recursive /usr/share/i18n/charmaps
2047         inst_recursive /usr/share/i18n/locales
2048         inst_recursive /usr/share/locale/en*
2049         inst_recursive /usr/share/locale/de*
2050         image_install /usr/share/locale/locale.alias
2051         # locale-gen might either generate each locale separately or merge them
2052         # into a single archive
2053         if ! (inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8 ||
2054               image_install /usr/lib/locale/locale-archive); then
2055             dfatal "Failed to install required locales"
2056             exit 1
2057         fi
2058     else
2059         inst_recursive /usr/lib/locale/C.*8 /usr/lib/locale/en_*8
2060     fi
2061 }
2062
2063 # shellcheck disable=SC2120
2064 install_keymaps() {
2065     local i p
2066     local -a prefix=(
2067         "/usr/lib"
2068         "/usr/share"
2069     )
2070
2071     dinfo "Install console keymaps"
2072
2073     if command -v meson >/dev/null \
2074             && [[ "$(meson configure "${BUILD_DIR:?}" | grep 'split-usr' | awk '{ print $2 }')" == "true" ]] \
2075             || [[ ! -L /lib ]]; then
2076         prefix+=(
2077             "/lib"
2078         )
2079     fi
2080
2081     if (( $# == 0 )); then
2082         for p in "${prefix[@]}"; do
2083             # The first three paths may be deprecated.
2084             # It seems now the last three paths are used by many distributions.
2085             for i in \
2086                 "$p"/kbd/keymaps/include/* \
2087                 "$p"/kbd/keymaps/i386/include/* \
2088                 "$p"/kbd/keymaps/i386/qwerty/us.* \
2089                 "$p"/kbd/keymaps/legacy/include/* \
2090                 "$p"/kbd/keymaps/legacy/i386/qwerty/us.* \
2091                 "$p"/kbd/keymaps/xkb/us*; do
2092                     [[ -f "$i" ]] || continue
2093                     inst "$i"
2094             done
2095         done
2096     else
2097         # When it takes any argument, then install more keymaps.
2098         for p in "${prefix[@]}"; do
2099             for i in \
2100                 "$p"/kbd/keymaps/include/* \
2101                 "$p"/kbd/keymaps/i386/*/* \
2102                 "$p"/kbd/keymaps/legacy/i386/*/* \
2103                 "$p"/kbd/keymaps/xkb/*; do
2104                     [[ -f "$i" ]] || continue
2105                     inst "$i"
2106             done
2107         done
2108     fi
2109 }
2110
2111 install_x11_keymaps() {
2112     dinfo "Install x11 keymaps"
2113
2114     if (( $# == 0 )); then
2115         # Install only keymap list.
2116         inst /usr/share/X11/xkb/rules/base.lst
2117     else
2118         # When it takes any argument, then install all keymaps.
2119         inst_recursive /usr/share/X11/xkb
2120     fi
2121 }
2122
2123 install_zoneinfo() {
2124     dinfo "Install time zones"
2125     inst_any /usr/share/zoneinfo/Asia/Seoul
2126     inst_any /usr/share/zoneinfo/Asia/Vladivostok
2127     inst_any /usr/share/zoneinfo/Australia/Sydney
2128     inst_any /usr/share/zoneinfo/Europe/Berlin
2129     inst_any /usr/share/zoneinfo/Europe/Dublin
2130     inst_any /usr/share/zoneinfo/Europe/Kiev
2131     inst_any /usr/share/zoneinfo/Pacific/Auckland
2132     inst_any /usr/share/zoneinfo/Pacific/Honolulu
2133     inst_any /usr/share/zoneinfo/CET
2134     inst_any /usr/share/zoneinfo/EET
2135     inst_any /usr/share/zoneinfo/UTC
2136 }
2137
2138 install_fonts() {
2139     dinfo "Install system fonts"
2140     for i in \
2141         /usr/lib/kbd/consolefonts/eurlatgr* \
2142         /usr/lib/kbd/consolefonts/latarcyrheb-sun16*; do
2143             [[ -f "$i" ]] || continue
2144             inst "$i"
2145     done
2146 }
2147
2148 install_terminfo() {
2149     dinfo "Install terminfo files"
2150     local terminfodir
2151     for terminfodir in /lib/terminfo /etc/terminfo /usr/share/terminfo; do
2152         [ -f "${terminfodir}/l/linux" ] && break
2153     done
2154     image_install -o "${terminfodir}/l/linux"
2155 }
2156
2157 has_user_dbus_socket() {
2158     if [ -f /usr/lib/systemd/user/dbus.socket ] || [ -f /etc/systemd/user/dbus.socket ]; then
2159         return 0
2160     else
2161         echo "Per-user instances are not supported. Skipping..."
2162         return 1
2163     fi
2164 }
2165
2166 setup_nspawn_root_hook() { :;}
2167
2168 setup_nspawn_root() {
2169     if [ -z "${initdir}" ]; then
2170         dfatal "\$initdir not defined"
2171         exit 1
2172     fi
2173
2174     rm -rf "${TESTDIR:?}/unprivileged-nspawn-root"
2175
2176     if get_bool "$RUN_IN_UNPRIVILEGED_CONTAINER"; then
2177         ddebug "cp -ar $initdir $TESTDIR/unprivileged-nspawn-root"
2178         cp -ar "$initdir" "$TESTDIR/unprivileged-nspawn-root"
2179     fi
2180
2181     setup_nspawn_root_hook
2182 }
2183
2184 setup_basic_dirs() {
2185     mkdir -p "${initdir:?}/run"
2186     mkdir -p "$initdir/etc/systemd/system"
2187     mkdir -p "$initdir/var/log/journal"
2188
2189
2190     for d in usr/bin usr/sbin bin etc lib "${libdir:?}" sbin tmp usr var var/log var/tmp dev proc sys sysroot root run run/lock run/initramfs; do
2191         if [ -L "/$d" ]; then
2192             inst_symlink "/$d"
2193         else
2194             inst_dir "/$d"
2195         fi
2196     done
2197
2198     ln -sfn /run "$initdir/var/run"
2199     ln -sfn /run/lock "$initdir/var/lock"
2200 }
2201
2202 mask_supporting_services() {
2203     # mask some services that we do not want to run in these tests
2204     ln -fsv /dev/null "${initdir:?}/etc/systemd/system/systemd-hwdb-update.service"
2205     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-journal-catalog-update.service"
2206     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.service"
2207     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-networkd.socket"
2208     ln -fsv /dev/null "$initdir/etc/systemd/system/systemd-resolved.service"
2209 }
2210
2211 inst_libs() {
2212     local bin="${1:?}"
2213     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2214     local file line
2215
2216     while read -r line; do
2217         [[ "$line" = 'not a dynamic executable' ]] && break
2218         # Ignore errors about our own stuff missing. This is most likely caused
2219         # by ldd attempting to use the unprefixed RPATH.
2220         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2221
2222         if [[ "$line" =~ $so_regex ]]; then
2223             file="${BASH_REMATCH[1]}"
2224             [[ -e "${initdir:?}/$file" ]] && continue
2225             inst_library "$file"
2226             continue
2227         fi
2228
2229         if [[ "$line" =~ not\ found ]]; then
2230             dfatal "Missing a shared library required by $bin."
2231             dfatal "Run \"ldd $bin\" to find out what it is."
2232             dfatal "$line"
2233             dfatal "Cannot create a test image."
2234             exit 1
2235         fi
2236     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2237 }
2238
2239 import_testdir() {
2240     # make sure we don't get a stale LOOPDEV value from old times
2241     local _LOOPDEV="${LOOPDEV:=}"
2242     # We don't want shellcheck to follow & check the $STATEFILE
2243     # shellcheck source=/dev/null
2244     [[ -e "$STATEFILE" ]] && . "$STATEFILE"
2245     LOOPDEV="$_LOOPDEV"
2246     if [[ ! -d "$TESTDIR" ]]; then
2247         if [[ -z "$TESTDIR" ]]; then
2248             TESTDIR="$(mktemp --tmpdir=$WORKDIR -d -t systemd-test.XXXXXX)"
2249         else
2250             mkdir -p "$TESTDIR"
2251         fi
2252
2253         cat >"$STATEFILE" <<EOF
2254 TESTDIR="$TESTDIR"
2255 EOF
2256         export TESTDIR
2257     fi
2258
2259     IMAGE_PRIVATE="${TESTDIR}/${IMAGE_NAME:?}.img"
2260     IMAGE_PUBLIC="${IMAGESTATEDIR:?}/${IMAGE_NAME}.img"
2261 }
2262
2263 import_initdir() {
2264     initdir="${TESTDIR:?}/root"
2265     mkdir -p "$initdir"
2266     export initdir
2267 }
2268
2269 get_cgroup_hierarchy() {
2270     case "$(stat -c '%T' -f /sys/fs/cgroup)" in
2271         cgroup2fs)
2272             echo "unified"
2273             ;;
2274         tmpfs)
2275             if [[ -d /sys/fs/cgroup/unified && "$(stat -c '%T' -f /sys/fs/cgroup/unified)" == cgroup2fs ]]; then
2276                 echo "hybrid"
2277             else
2278                 echo "legacy"
2279             fi
2280             ;;
2281         *)
2282             dfatal "Failed to determine host's cgroup hierarchy"
2283             exit 1
2284     esac
2285 }
2286
2287 ## @brief Converts numeric logging level to the first letter of level name.
2288 #
2289 # @param lvl Numeric logging level in range from 1 to 6.
2290 # @retval 1 if @a lvl is out of range.
2291 # @retval 0 if @a lvl is correct.
2292 # @result Echoes first letter of level name.
2293 _lvl2char() {
2294     case "$1" in
2295         1) echo F;;
2296         2) echo E;;
2297         3) echo W;;
2298         4) echo I;;
2299         5) echo D;;
2300         6) echo T;;
2301         *) return 1;;
2302     esac
2303 }
2304
2305 ## @brief Internal helper function for _do_dlog()
2306 #
2307 # @param lvl Numeric logging level.
2308 # @param msg Message.
2309 # @retval 0 It's always returned, even if logging failed.
2310 #
2311 # @note This function is not supposed to be called manually. Please use
2312 # dtrace(), ddebug(), or others instead which wrap this one.
2313 #
2314 # This function calls _do_dlog() either with parameter msg, or if
2315 # none is given, it will read standard input and will use every line as
2316 # a message.
2317 #
2318 # This enables:
2319 # dwarn "This is a warning"
2320 # echo "This is a warning" | dwarn
2321 LOG_LEVEL="${LOG_LEVEL:-4}"
2322
2323 dlog() {
2324     local lvl lvlc
2325
2326     [ -z "$LOG_LEVEL" ] && return 0
2327     lvl="${1:?}"; shift
2328     [ "$lvl" -le "$LOG_LEVEL" ] || return 0
2329     lvlc="$(_lvl2char "$lvl")" || return 0
2330
2331     if [ $# -ge 1 ]; then
2332         echo "$lvlc: $*"
2333     else
2334         while read -r line; do
2335             echo "$lvlc: " "$line"
2336         done
2337     fi
2338 }
2339
2340 ## @brief Logs message at TRACE level (6)
2341 #
2342 # @param msg Message.
2343 # @retval 0 It's always returned, even if logging failed.
2344 dtrace() {
2345     set +x
2346     dlog 6 "$@"
2347     if get_bool "${debug:=}"; then
2348         set -x
2349     fi
2350 }
2351
2352 ## @brief Logs message at DEBUG level (5)
2353 #
2354 # @param msg Message.
2355 # @retval 0 It's always returned, even if logging failed.
2356 ddebug() {
2357     dlog 5 "$@"
2358 }
2359
2360 ## @brief Logs message at INFO level (4)
2361 #
2362 # @param msg Message.
2363 # @retval 0 It's always returned, even if logging failed.
2364 dinfo() {
2365     set +x
2366     dlog 4 "$@"
2367     if get_bool "${debug:=}"; then
2368         set -x
2369     fi
2370 }
2371
2372 ## @brief Logs message at WARN level (3)
2373 #
2374 # @param msg Message.
2375 # @retval 0 It's always returned, even if logging failed.
2376 dwarn() {
2377     set +x
2378     dlog 3 "$@"
2379     if get_bool "${debug:=}"; then
2380         set -x
2381     fi
2382 }
2383
2384 ## @brief Logs message at ERROR level (2)
2385 #
2386 # @param msg Message.
2387 # @retval 0 It's always returned, even if logging failed.
2388 derror() {
2389     dlog 2 "$@"
2390 }
2391
2392 ## @brief Logs message at FATAL level (1)
2393 #
2394 # @param msg Message.
2395 # @retval 0 It's always returned, even if logging failed.
2396 dfatal() {
2397     set +x
2398     dlog 1 "$@"
2399     if get_bool "${debug:=}"; then
2400         set -x
2401     fi
2402 }
2403
2404
2405 # Generic substring function.  If $2 is in $1, return 0.
2406 strstr() { [ "${1#*"$2"*}" != "$1" ]; }
2407
2408 # normalize_path <path>
2409 # Prints the normalized path, where it removes any duplicated
2410 # and trailing slashes.
2411 # Example:
2412 # $ normalize_path ///test/test//
2413 # /test/test
2414 normalize_path() {
2415     shopt -q -s extglob
2416     set -- "${1//+(\/)//}"
2417     shopt -q -u extglob
2418     echo "${1%/}"
2419 }
2420
2421 # convert_abs_rel <from> <to>
2422 # Prints the relative path, when creating a symlink to <to> from <from>.
2423 # Example:
2424 # $ convert_abs_rel /usr/bin/test /bin/test-2
2425 # ../../bin/test-2
2426 # $ ln -s $(convert_abs_rel /usr/bin/test /bin/test-2) /usr/bin/test
2427 convert_abs_rel() {
2428     local __current __absolute __abssize __cursize __newpath
2429     local -i __i __level
2430
2431     set -- "$(normalize_path "${1:?}")" "$(normalize_path "${2:?}")"
2432
2433     # corner case #1 - self looping link
2434     [[ "$1" == "$2" ]] && { echo "${1##*/}"; return; }
2435
2436     # corner case #2 - own dir link
2437     [[ "${1%/*}" == "$2" ]] && { echo "."; return; }
2438
2439     IFS="/" read -ra __current <<< "$1"
2440     IFS="/" read -ra __absolute <<< "$2"
2441
2442     __abssize=${#__absolute[@]}
2443     __cursize=${#__current[@]}
2444
2445     while [[ "${__absolute[__level]}" == "${__current[__level]}" ]]; do
2446         (( __level++ ))
2447         if (( __level > __abssize || __level > __cursize ))
2448         then
2449             break
2450         fi
2451     done
2452
2453     for ((__i = __level; __i < __cursize-1; __i++)); do
2454         if ((__i > __level))
2455         then
2456             __newpath=$__newpath"/"
2457         fi
2458         __newpath=$__newpath".."
2459     done
2460
2461     for ((__i = __level; __i < __abssize; __i++)); do
2462         if [[ -n $__newpath ]]
2463         then
2464             __newpath=$__newpath"/"
2465         fi
2466         __newpath=$__newpath${__absolute[__i]}
2467     done
2468
2469     echo "$__newpath"
2470 }
2471
2472
2473 # Install a directory, keeping symlinks as on the original system.
2474 # Example: if /lib points to /lib64 on the host, "inst_dir /lib/file"
2475 # will create ${initdir}/lib64, ${initdir}/lib64/file,
2476 # and a symlink ${initdir}/lib -> lib64.
2477 inst_dir() {
2478     local dir="${1:?}"
2479     local part="${dir%/*}"
2480     local file
2481
2482     [[ -e "${initdir:?}/${dir}" ]] && return 0  # already there
2483
2484     while [[ "$part" != "${part%/*}" ]] && ! [[ -e "${initdir}/${part}" ]]; do
2485         dir="$part $dir"
2486         part="${part%/*}"
2487     done
2488
2489     # iterate over parent directories
2490     for file in $dir; do
2491         [[ -e "${initdir}/$file" ]] && continue
2492         if [[ -L $file ]]; then
2493             inst_symlink "$file"
2494         else
2495             # create directory
2496             mkdir -m 0755 "${initdir}/$file" || return 1
2497             [[ -e "$file" ]] && chmod --reference="$file" "${initdir}/$file"
2498             chmod u+w "${initdir}/$file"
2499         fi
2500     done
2501 }
2502
2503 # $1 = file to copy to ramdisk
2504 # $2 (optional) Name for the file on the ramdisk
2505 # Location of the image dir is assumed to be $initdir
2506 # We never overwrite the target if it exists.
2507 inst_simple() {
2508     [[ -f "${1:?}" ]] || return 1
2509     strstr "$1" "/" || return 1
2510
2511     local src="$1"
2512     local target="${2:-$1}"
2513     if ! [[ -d ${initdir:?}/$target ]]; then
2514         [[ -e ${initdir}/$target ]] && return 0
2515         [[ -L ${initdir}/$target ]] && return 0
2516         [[ -d "${initdir}/${target%/*}" ]] || inst_dir "${target%/*}"
2517     fi
2518     # install checksum files also
2519     if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2520         inst "${src%/*}/.${src##*/}.hmac" "${target%/*}/.${target##*/}.hmac"
2521     fi
2522     ddebug "Installing $src"
2523     cp --sparse=always -pfL "$src" "${initdir}/$target"
2524 }
2525
2526 # find symlinks linked to given library file
2527 # $1 = library file
2528 # Function searches for symlinks by stripping version numbers appended to
2529 # library filename, checks if it points to the same target and finally
2530 # prints the list of symlinks to stdout.
2531 #
2532 # Example:
2533 # rev_lib_symlinks libfoo.so.8.1
2534 # output: libfoo.so.8 libfoo.so
2535 # (Only if libfoo.so.8 and libfoo.so exists on host system.)
2536 rev_lib_symlinks() {
2537     local fn="${1:?}"
2538     local links=""
2539     local orig
2540     orig="$(readlink -f "$1")"
2541
2542     [[ "${fn}" =~ .*\.so\..* ]] || return 1
2543
2544     until [[ "${fn##*.}" == so ]]; do
2545         fn="${fn%.*}"
2546         [[ -L "${fn}" && "$(readlink -f "${fn}")" == "${orig}" ]] && links+=" ${fn}"
2547     done
2548
2549     echo "${links}"
2550 }
2551
2552 # Same as above, but specialized to handle dynamic libraries.
2553 # It handles making symlinks according to how the original library
2554 # is referenced.
2555 inst_library() {
2556     local src="${1:?}"
2557     local dest="${2:-$1}"
2558     local reallib symlink
2559
2560     strstr "$1" "/" || return 1
2561     [[ -e ${initdir:?}/$dest ]] && return 0
2562     if [[ -L $src ]]; then
2563         # install checksum files also
2564         if [[ -e "${src%/*}/.${src##*/}.hmac" ]]; then
2565             inst "${src%/*}/.${src##*/}.hmac" "${dest%/*}/.${dest##*/}.hmac"
2566         fi
2567         reallib="$(readlink -f "$src")"
2568         inst_simple "$reallib" "$reallib"
2569         inst_dir "${dest%/*}"
2570         [[ -d "${dest%/*}" ]] && dest="$(readlink -f "${dest%/*}")/${dest##*/}"
2571         ddebug "Creating symlink $reallib -> $dest"
2572         ln -sfn -- "$(convert_abs_rel "${dest}" "${reallib}")" "${initdir}/${dest}"
2573     else
2574         inst_simple "$src" "$dest"
2575     fi
2576
2577     # Create additional symlinks.  See rev_symlinks description.
2578     for symlink in $(rev_lib_symlinks "$src") ${reallib:+$(rev_lib_symlinks "$reallib")}; do
2579         if [[ ! -e "$initdir/$symlink" ]]; then
2580             ddebug "Creating extra symlink: $symlink"
2581             inst_symlink "$symlink"
2582         fi
2583     done
2584 }
2585
2586 # find a binary.  If we were not passed the full path directly,
2587 # search in the usual places to find the binary.
2588 find_binary() {
2589     local bin="${1:?}"
2590     if [[ -z ${bin##/*} ]]; then
2591         if [[ -x "$bin" ]] || { strstr "$bin" ".so" && ldd "$bin" &>/dev/null; }; then
2592             echo "$bin"
2593             return 0
2594         fi
2595     fi
2596
2597     type -P "$bin"
2598 }
2599
2600 # Same as above, but specialized to install binary executables.
2601 # Install binary executable, and all shared library dependencies, if any.
2602 inst_binary() {
2603     local bin="${1:?}"
2604     local path target
2605
2606     # In certain cases we might attempt to install a binary which is already
2607     # present in the test image, yet it's missing from the host system.
2608     # In such cases, let's check if the binary indeed exists in the image
2609     # before doing any other checks. If it does, immediately return with
2610     # success.
2611     if [[ $# -eq 1 ]]; then
2612         for path in "" bin sbin usr/bin usr/sbin; do
2613             [[ -e "${initdir:?}${path:+/$path}/${bin}" ]] && return 0
2614         done
2615     fi
2616
2617     bin="$(find_binary "$bin")" || return 1
2618     target="${2:-$bin}"
2619     [[ -e "${initdir:?}/$target" ]] && return 0
2620     [[ -L "$bin" ]] && inst_symlink "$bin" "$target" && return 0
2621
2622     local file line
2623     local so_regex='([^ ]*/lib[^/]*/[^ ]*\.so[^ ]*)'
2624     # DSOs provided by systemd
2625     local systemd_so_regex='/(libudev|libsystemd.*|.+[\-_]systemd([\-_].+)?|libnss_(mymachines|myhostname|resolve)).so'
2626     local wrap_binary=0
2627     # I love bash!
2628     while read -r line; do
2629         [[ "$line" = 'not a dynamic executable' ]] && break
2630
2631         # Ignore errors about our own stuff missing. This is most likely caused
2632         # by ldd attempting to use the unprefixed RPATH.
2633         [[ "$line" =~ libsystemd.*\ not\ found ]] && continue
2634
2635         # We're built with ASan and the target binary loads one of the systemd's
2636         # DSOs, so we need to tweak the environment before executing the binary
2637         if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$line" =~ $systemd_so_regex ]]; then
2638             wrap_binary=1
2639         fi
2640
2641         if [[ "$line" =~ $so_regex ]]; then
2642             file="${BASH_REMATCH[1]}"
2643             [[ -e "${initdir}/$file" ]] && continue
2644             inst_library "$file"
2645             continue
2646         fi
2647
2648         if [[ "$line" =~ not\ found ]]; then
2649             dfatal "Missing a shared library required by $bin."
2650             dfatal "Run \"ldd $bin\" to find out what it is."
2651             dfatal "$line"
2652             dfatal "Cannot create a test image."
2653             exit 1
2654         fi
2655     done < <(LC_ALL=C ldd "$bin" 2>/dev/null)
2656
2657     # Same as above, but we need to wrap certain libraries unconditionally
2658     #
2659     # chown, getent, login, setfacl, su, useradd, userdel
2660     #    - dlopen() (not only) systemd's PAM modules
2661     # ls, mkfs.*, mksquashfs, mkswap, setpriv, stat
2662     #   - pull in nss_systemd with certain options (like ls -l) when
2663     #     nsswitch.conf uses [SUCCESS=merge] (like on Arch Linux)
2664     # delv, dig - pull in nss_resolve if `resolve` is in nsswitch.conf
2665     # tar - called by machinectl in TEST-25
2666     bin_rx='/(chown|delv|dig|getent|login|ls|mkfs\.[a-z0-9]+|mksquashfs|mkswap|setfacl|setpriv|stat|su|tar|useradd|userdel)$'
2667     if get_bool "$IS_BUILT_WITH_ASAN" && [[ "$bin" =~ $bin_rx ]]; then
2668         wrap_binary=1
2669     fi
2670
2671     # If the target binary is built with ASan support, we don't need to wrap
2672     # it, as it should handle everything by itself
2673     if get_bool "$wrap_binary" && ! is_built_with_asan "$bin"; then
2674         dinfo "Creating ASan-compatible wrapper for binary '$target'"
2675         # Install the target binary with a ".orig" suffix
2676         inst_simple "$bin" "${target}.orig"
2677         # Create a simple shell wrapper in place of the target binary, which
2678         # sets necessary ASan-related env variables and then exec()s the
2679         # suffixed target binary
2680         cat >"$initdir/$target" <<EOF
2681 #!/bin/bash
2682 # Preload the ASan runtime DSO, otherwise ASAn will complain
2683 export LD_PRELOAD="$ASAN_RT_PATH"
2684 # Disable LSan to speed things up, since we don't care about leak reports
2685 # from 'external' binaries
2686 export ASAN_OPTIONS=detect_leaks=0
2687 # Set argv[0] to the original binary name without the ".orig" suffix
2688 exec -a "\$0" -- "${target}.orig" "\$@"
2689 EOF
2690         chmod +x "$initdir/$target"
2691     else
2692         inst_simple "$bin" "$target"
2693     fi
2694 }
2695
2696 # same as above, except for shell scripts.
2697 # If your shell script does not start with shebang, it is not a shell script.
2698 inst_script() {
2699     local bin line shebang_regex
2700     bin="$(find_binary "${1:?}")" || return 1
2701     shift
2702
2703     read -r -n 80 line <"$bin"
2704     # If debug is set, clean unprintable chars to prevent messing up the term
2705     get_bool "${debug:=}" && line="$(echo -n "$line" | tr -c -d '[:print:][:space:]')"
2706     shebang_regex='(#! *)(/[^ ]+).*'
2707     [[ "$line" =~ $shebang_regex ]] || return 1
2708     inst "${BASH_REMATCH[2]}" && inst_simple "$bin" "$@"
2709 }
2710
2711 # same as above, but specialized for symlinks
2712 inst_symlink() {
2713     local src="${1:?}"
2714     local target="${2:-$src}"
2715     local realsrc
2716
2717     strstr "$src" "/" || return 1
2718     [[ -L "$src" ]] || return 1
2719     [[ -L "${initdir:?}/$target" ]] && return 0
2720     realsrc="$(readlink -f "$src")"
2721     if ! [[ -e "$initdir/$realsrc" ]]; then
2722         if [[ -d "$realsrc" ]]; then
2723             inst_dir "$realsrc"
2724         else
2725             inst "$realsrc"
2726         fi
2727     fi
2728     [[ ! -e "$initdir/${target%/*}" ]] && inst_dir "${target%/*}"
2729     [[ -d "${target%/*}" ]] && target="$(readlink -f "${target%/*}")/${target##*/}"
2730     ln -sfn -- "$(convert_abs_rel "${target}" "${realsrc}")" "$initdir/$target"
2731 }
2732
2733 # attempt to install any programs specified in a udev rule
2734 inst_rule_programs() {
2735     local rule="${1:?}"
2736     local prog bin
2737
2738     sed -rn 's/^.*?PROGRAM==?"([^ "]+).*$/\1/p' "$rule" | while read -r prog; do
2739         if [ -x "/lib/udev/$prog" ]; then
2740             bin="/lib/udev/$prog"
2741         else
2742             if ! bin="$(find_binary "$prog")"; then
2743                 dinfo "Skipping program $prog used in udev rule $(basename "$rule") as it cannot be found"
2744                 continue
2745             fi
2746         fi
2747
2748         #dinfo "Installing $_bin due to it's use in the udev rule $(basename $1)"
2749         image_install "$bin"
2750     done
2751 }
2752
2753 # udev rules always get installed in the same place, so
2754 # create a function to install them to make life simpler.
2755 inst_rules() {
2756     local target=/etc/udev/rules.d
2757     local found rule
2758
2759     inst_dir "/lib/udev/rules.d"
2760     inst_dir "$target"
2761     for rule in "$@"; do
2762         if [ "${rule#/}" = "$rule" ]; then
2763             for r in /lib/udev/rules.d /etc/udev/rules.d; do
2764                 if [[ -f "$r/$rule" ]]; then
2765                     found="$r/$rule"
2766                     inst_simple "$found"
2767                     inst_rule_programs "$found"
2768                 fi
2769             done
2770         fi
2771         for r in '' ./; do
2772             if [[ -f "${r}${rule}" ]]; then
2773                 found="${r}${rule}"
2774                 inst_simple "$found" "$target/${found##*/}"
2775                 inst_rule_programs "$found"
2776             fi
2777         done
2778         [[ $found ]] || dinfo "Skipping udev rule: $rule"
2779         found=
2780     done
2781 }
2782
2783 # general purpose installation function
2784 # Same args as above.
2785 inst() {
2786     case $# in
2787         1) ;;
2788         2)
2789             [[ ! "$initdir" && -d "$2" ]] && export initdir="$2"
2790             [[ "$initdir" = "$2" ]] && set "$1"
2791             ;;
2792         3)
2793             [[ -z "$initdir" ]] && export initdir="$2"
2794             set "$1" "$3"
2795             ;;
2796         *)
2797             dfatal "inst only takes 1 or 2 or 3 arguments"
2798             exit 1
2799             ;;
2800     esac
2801
2802     local fun
2803     for fun in inst_symlink inst_script inst_binary inst_simple; do
2804         "$fun" "$@" && return 0
2805     done
2806
2807     dwarn "Failed to install '$1'"
2808     return 1
2809 }
2810
2811 # install any of listed files
2812 #
2813 # If first argument is '-d' and second some destination path, first accessible
2814 # source is installed into this path, otherwise it will installed in the same
2815 # path as source.  If none of listed files was installed, function return 1.
2816 # On first successful installation it returns with 0 status.
2817 #
2818 # Example:
2819 #
2820 # inst_any -d /bin/foo /bin/bar /bin/baz
2821 #
2822 # Lets assume that /bin/baz exists, so it will be installed as /bin/foo in
2823 # initrd.
2824 inst_any() {
2825     local dest file
2826
2827     [[ "${1:?}" = '-d' ]] && dest="${2:?}" && shift 2
2828
2829     for file in "$@"; do
2830         if [[ -e "$file" ]]; then
2831             [[ -n "$dest" ]] && inst "$file" "$dest" && return 0
2832             inst "$file" && return 0
2833         fi
2834     done
2835
2836     return 1
2837 }
2838
2839 inst_recursive() {
2840     local p item
2841
2842     for p in "$@"; do
2843         # Make sure the source exists, as the process substitution below
2844         # suppresses errors
2845         stat "$p" >/dev/null || return 1
2846
2847         while read -r item; do
2848             if [[ -d "$item" ]]; then
2849                 inst_dir "$item"
2850             elif [[ -f "$item" ]]; then
2851                 inst_simple "$item"
2852             fi
2853         done < <(find "$p" 2>/dev/null)
2854     done
2855 }
2856
2857 # image_install [-o ] <file> [<file> ... ]
2858 # Install <file> to the test image
2859 # -o optionally install the <file> and don't fail, if it is not there
2860 image_install() {
2861     local optional=no
2862     local prog="${1:?}"
2863
2864     if [[ "$prog" = '-o' ]]; then
2865         optional=yes
2866         shift
2867     fi
2868
2869     for prog in "$@"; do
2870         if ! inst "$prog" ; then
2871             if get_bool "$optional"; then
2872                 dinfo "Skipping program $prog as it cannot be found and is" \
2873                     "flagged to be optional"
2874             else
2875                 dfatal "Failed to install $prog"
2876                 exit 1
2877             fi
2878         fi
2879     done
2880 }
2881
2882 # Install a single kernel module along with any firmware it may require.
2883 # $1 = full path to kernel module to install
2884 install_kmod_with_fw() {
2885     local module="${1:?}"
2886     # no need to go further if the module is already installed
2887     [[ -e "${initdir:?}/lib/modules/${KERNEL_VER:?}/${module##*"/lib/modules/$KERNEL_VER/"}" ]] && return 0
2888     [[ -e "$initdir/.kernelmodseen/${module##*/}" ]] && return 0
2889
2890     [ -d "$initdir/.kernelmodseen" ] && : >"$initdir/.kernelmodseen/${module##*/}"
2891
2892     inst_simple "$module" "/lib/modules/$KERNEL_VER/${module##*"/lib/modules/$KERNEL_VER/"}" || return $?
2893
2894     local modname="${module##*/}"
2895     local fwdir found fw
2896     modname="${modname%.ko*}"
2897
2898     while read -r fw; do
2899         found=
2900         for fwdir in /lib/firmware/updates /lib/firmware; do
2901             if [[ -d "$fwdir" && -f "$fwdir/$fw" ]]; then
2902                 inst_simple "$fwdir/$fw" "/lib/firmware/$fw"
2903                 found=yes
2904             fi
2905         done
2906         if ! get_bool "$found"; then
2907             if ! grep -qe "\<${modname//-/_}\>" /proc/modules; then
2908                 dinfo "Possible missing firmware \"${fw}\" for kernel module" \
2909                     "\"${modname}.ko\""
2910             else
2911                 dwarn "Possible missing firmware \"${fw}\" for kernel module" \
2912                     "\"${modname}.ko\""
2913             fi
2914         fi
2915     done < <(modinfo -k "$KERNEL_VER" -F firmware "$module" 2>/dev/null)
2916     return 0
2917 }
2918
2919 # Do something with all the dependencies of a kernel module.
2920 # Note that kernel modules depend on themselves using the technique we use
2921 # $1 = function to call for each dependency we find
2922 #      It will be passed the full path to the found kernel module
2923 # $2 = module to get dependencies for
2924 # rest of args = arguments to modprobe
2925 for_each_kmod_dep() {
2926     local func="${1:?}"
2927     local kmod="${2:?}"
2928     local found=0
2929     local cmd modpath
2930     shift 2
2931
2932     while read -r cmd modpath _; do
2933         [[ "$cmd" = insmod ]] || continue
2934         "$func" "$modpath" || return $?
2935         found=1
2936     done < <(modprobe "$@" --ignore-install --show-depends "$kmod")
2937
2938     ! get_bool "$found" && return 1
2939     return 0
2940 }
2941
2942 # instmods [-c] <kernel module> [<kernel module> ... ]
2943 # instmods [-c] <kernel subsystem>
2944 # install kernel modules along with all their dependencies.
2945 # <kernel subsystem> can be e.g. "=block" or "=drivers/usb/storage"
2946 # FIXME(?): dracutdevs/dracut@f4e38c0da8d6bf3764c1ad753d9d52aef63050e5
2947 instmods() {
2948     local check=no
2949     if [[ $# -ge 0 && "$1" = '-c' ]]; then
2950         check=yes
2951         shift
2952     fi
2953
2954     inst1mod() {
2955         local mod="${1:?}"
2956         local ret=0
2957         local mod_dir="/lib/modules/${KERNEL_VER:?}/"
2958
2959         case "$mod" in
2960             =*)
2961                 if [ -f "${mod_dir}/modules.${mod#=}" ]; then
2962                     (
2963                         [[ "$mpargs" ]] && echo "$mpargs"
2964                         cat "${mod_dir}/modules.${mod#=}"
2965                     ) | instmods
2966                 else
2967                     (
2968                         [[ "$mpargs" ]] && echo "$mpargs"
2969                         find "$mod_dir" -path "*/${mod#=}/*" -name "*.ko*" -type f -printf '%f\n'
2970                     ) | instmods
2971                 fi
2972                 ;;
2973             --*)
2974                 mpargs+=" $mod"
2975                 ;;
2976             i2o_scsi)
2977                 # Do not load this diagnostic-only module
2978                 return
2979                 ;;
2980             *)
2981                 mod=${mod##*/}
2982                 # if we are already installed, skip this module and go on
2983                 # to the next one.
2984                 [[ -f "${initdir:?}/.kernelmodseen/${mod%.ko}.ko" ]] && return
2985
2986                 # We use '-d' option in modprobe only if modules prefix path
2987                 # differs from default '/'.  This allows us to use Dracut with
2988                 # old version of modprobe which doesn't have '-d' option.
2989                 local mod_dirname=${mod_dir%%/lib/modules/*}
2990                 [[ -n ${mod_dirname} ]] && mod_dirname="-d ${mod_dirname}/"
2991
2992                 # ok, load the module, all its dependencies, and any firmware
2993                 # it may require
2994                 for_each_kmod_dep install_kmod_with_fw "$mod" \
2995                     --set-version "$KERNEL_VER" \
2996                     ${mod_dirname:+"$mod_dirname"} \
2997                     ${mpargs:+"$mpargs"}
2998                 ((ret+=$?))
2999                 ;;
3000         esac
3001         return "$ret"
3002     }
3003
3004     local mod mpargs
3005
3006     if [[ $# -eq 0 ]]; then  # filenames from stdin
3007         while read -r mod; do
3008             if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3009                 dfatal "Failed to install $mod"
3010                 return 1
3011             fi
3012         done
3013     fi
3014
3015     for mod in "$@"; do # filenames as arguments
3016         if ! inst1mod "${mod%.ko*}" && [ "$check" = "yes" ]; then
3017             dfatal "Failed to install $mod"
3018             return 1
3019         fi
3020     done
3021
3022     return 0
3023 }
3024
3025 _umount_dir() {
3026     local mountpoint="${1:?}"
3027     if mountpoint -q "$mountpoint"; then
3028         ddebug "umount $mountpoint"
3029         umount "$mountpoint"
3030     fi
3031 }
3032
3033 # can be overridden in specific test
3034 test_setup_cleanup() {
3035     cleanup_initdir
3036 }
3037
3038 _test_cleanup() {
3039     # (post-test) cleanup should always ignore failure and cleanup as much as possible
3040     (
3041         set +e
3042         [[ -n "$initdir" ]] && _umount_dir "$initdir"
3043         [[ -n "$IMAGE_PUBLIC" ]] && rm -vf "$IMAGE_PUBLIC"
3044         # If multiple setups/cleans are ran in parallel, this can cause a race
3045         if [[ -n "$IMAGESTATEDIR" && $TEST_PARALLELIZE -ne 1 ]]; then
3046             rm -vf "${IMAGESTATEDIR}/default.img"
3047         fi
3048         [[ -n "$TESTDIR" ]] && rm -vfr "$TESTDIR"
3049         [[ -n "$STATEFILE" ]] && rm -vf "$STATEFILE"
3050     ) || :
3051 }
3052
3053 # can be overridden in specific test
3054 test_cleanup() {
3055     _test_cleanup
3056 }
3057
3058 test_cleanup_again() {
3059     [ -n "$TESTDIR" ] || return
3060     rm -rf "$TESTDIR/unprivileged-nspawn-root"
3061     [[ -n "$initdir" ]] && _umount_dir "$initdir"
3062 }
3063
3064 test_create_image() {
3065     create_empty_image_rootdir
3066
3067     # Create what will eventually be our root filesystem onto an overlay
3068     (
3069         LOG_LEVEL=5
3070         setup_basic_environment
3071     )
3072 }
3073
3074 test_setup() {
3075     if get_bool "${TEST_REQUIRE_INSTALL_TESTS:?}" && \
3076             command -v meson >/dev/null && \
3077             [[ "$(meson configure "${BUILD_DIR:?}" | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
3078         dfatal "$BUILD_DIR needs to be built with -Dinstall-tests=true"
3079         exit 1
3080     fi
3081
3082     if [ -e "${IMAGE_PRIVATE:?}" ]; then
3083         echo "Reusing existing image $IMAGE_PRIVATE → $(realpath "$IMAGE_PRIVATE")"
3084         mount_initdir
3085     else
3086         if [ ! -e "${IMAGE_PUBLIC:?}" ]; then
3087             # default.img is the base that every test uses and optionally appends to
3088             if [ ! -e "${IMAGESTATEDIR:?}/default.img" ] || [ -n "${TEST_FORCE_NEWIMAGE:=}" ]; then
3089                 # Create the backing public image, but then completely unmount
3090                 # it and drop the loopback device responsible for it, since we're
3091                 # going to symlink/copy the image and mount it again from
3092                 # elsewhere.
3093                 local image_old="${IMAGE_PUBLIC}"
3094                 if [ -z "${TEST_FORCE_NEWIMAGE}" ]; then
3095                     IMAGE_PUBLIC="${IMAGESTATEDIR}/default.img"
3096                 fi
3097                 test_create_image
3098                 test_setup_cleanup
3099                 umount_loopback
3100                 cleanup_loopdev
3101                 IMAGE_PUBLIC="${image_old}"
3102             fi
3103             if [ "${IMAGE_NAME:?}" != "default" ] && ! get_bool "${TEST_FORCE_NEWIMAGE}"; then
3104                 cp -v "$(realpath "${IMAGESTATEDIR}/default.img")" "$IMAGE_PUBLIC"
3105             fi
3106         fi
3107
3108         local hook_defined
3109         declare -f -F test_append_files >/dev/null && hook_defined=yes || hook_defined=no
3110
3111         echo "Reusing existing cached image $IMAGE_PUBLIC → $(realpath "$IMAGE_PUBLIC")"
3112         if get_bool "$TEST_PARALLELIZE" || get_bool "$hook_defined"; then
3113             cp -v -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3114         else
3115             ln -sv -- "$(realpath "$IMAGE_PUBLIC")" "$IMAGE_PRIVATE"
3116         fi
3117
3118         mount_initdir
3119
3120         if get_bool "${TEST_SUPPORTING_SERVICES_SHOULD_BE_MASKED}"; then
3121             dinfo "Masking supporting services"
3122             mask_supporting_services
3123         fi
3124
3125         # Send stdout/stderr of testsuite-*.service units to both journal and
3126         # console to make debugging in CIs easier
3127         # Note: we can't use a dropin for `testsuite-.service`, since that also
3128         #       overrides 'sub-units' of some tests that already use a specific
3129         #       value for Standard(Output|Error)=
3130         #       (e.g. test/units/testsuite-66-deviceisolation.service)
3131         if ! get_bool "$INTERACTIVE_DEBUG"; then
3132             local dropin_dir="${initdir:?}/etc/systemd/system/testsuite-${TESTID:?}.service.d"
3133             mkdir -p "$dropin_dir"
3134             printf '[Service]\nStandardOutput=journal+console\nStandardError=journal+console' >"$dropin_dir/99-stdout.conf"
3135         fi
3136
3137         if get_bool "$IS_BUILT_WITH_COVERAGE"; then
3138             # Do an initial coverage capture, to make sure the final report includes
3139             # files that the tests didn't touch at all
3140             lcov --initial --capture --directory "${initdir}/${BUILD_DIR:?}" --output-file "${TESTDIR:?}/coverage-base"
3141         fi
3142
3143         if get_bool "$hook_defined"; then
3144             test_append_files "${initdir:?}"
3145         fi
3146     fi
3147
3148     setup_nspawn_root
3149 }
3150
3151 test_run() {
3152     local test_id="${1:?}"
3153     mount_initdir
3154
3155     if ! get_bool "${TEST_NO_QEMU:=}"; then
3156         if run_qemu "$test_id"; then
3157             check_result_qemu || { echo "qemu test failed"; return 1; }
3158         else
3159             dwarn "can't run qemu, skipping"
3160         fi
3161     fi
3162     if ! get_bool "${TEST_NO_NSPAWN:=}"; then
3163         mount_initdir
3164         if run_nspawn "${initdir:?}" "$test_id"; then
3165             check_result_nspawn "$initdir" || { echo "nspawn-root test failed"; return 1; }
3166         else
3167             dwarn "can't run systemd-nspawn, skipping"
3168         fi
3169
3170         if get_bool "${RUN_IN_UNPRIVILEGED_CONTAINER:=}"; then
3171             dir="$TESTDIR/unprivileged-nspawn-root"
3172             if NSPAWN_ARGUMENTS="-U --private-network ${NSPAWN_ARGUMENTS:-}" run_nspawn "$dir" "$test_id"; then
3173                 check_result_nspawn "$dir" || { echo "unprivileged-nspawn-root test failed"; return 1; }
3174             else
3175                 dwarn "can't run systemd-nspawn, skipping"
3176             fi
3177         fi
3178     fi
3179     return 0
3180 }
3181
3182 do_test() {
3183     if [[ $UID != "0" ]]; then
3184         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: not root" >&2
3185         exit 0
3186     fi
3187
3188     if get_bool "${TEST_NO_QEMU:=}" && get_bool "${TEST_NO_NSPAWN:=}"; then
3189         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: both qemu and nspawn disabled" >&2
3190         exit 0
3191     fi
3192
3193     if get_bool "${TEST_QEMU_ONLY:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3194         echo "TEST: $TEST_DESCRIPTION [SKIPPED]: qemu-only tests requested" >&2
3195         exit 0
3196     fi
3197
3198     if get_bool "${TEST_PREFER_NSPAWN:=}" && ! get_bool "$TEST_NO_NSPAWN"; then
3199         TEST_NO_QEMU=1
3200     fi
3201
3202     # Detect lib paths
3203     [[ "$libdir" ]] || for libdir in /lib64 /lib; do
3204         [[ -d $libdir ]] && libdirs+=" $libdir" && break
3205     done
3206
3207     [[ "$usrlibdir" ]] || for usrlibdir in /usr/lib64 /usr/lib; do
3208         [[ -d $usrlibdir ]] && libdirs+=" $usrlibdir" && break
3209     done
3210
3211     mkdir -p "$WORKDIR"
3212     mkdir -p "$STATEDIR"
3213
3214     import_testdir
3215     import_initdir
3216
3217     if [ -n "${SUDO_USER}" ]; then
3218         ddebug "Making ${TESTDIR:?} readable for ${SUDO_USER} (acquired from sudo)"
3219         setfacl -m "user:${SUDO_USER:?}:r-X" "${TESTDIR:?}"
3220     fi
3221
3222     testname="$(basename "$PWD")"
3223
3224     while (($# > 0)); do
3225         case $1 in
3226             --run)
3227                 echo "${testname} RUN: $TEST_DESCRIPTION"
3228                 test_run "$TESTID"
3229                 ret=$?
3230                 if [ $ret -eq 0 ]; then
3231                     echo "${testname} RUN: $TEST_DESCRIPTION [OK]"
3232                 else
3233                     echo "${testname} RUN: $TEST_DESCRIPTION [FAILED]"
3234                 fi
3235                 exit $ret
3236                 ;;
3237             --setup)
3238                 echo "${testname} SETUP: $TEST_DESCRIPTION"
3239                 test_setup
3240                 test_setup_cleanup
3241                 ;;
3242             --clean)
3243                 echo "${testname} CLEANUP: $TEST_DESCRIPTION"
3244                 test_cleanup
3245                 ;;
3246             --clean-again)
3247                 echo "${testname} CLEANUP AGAIN: $TEST_DESCRIPTION"
3248                 test_cleanup_again
3249                 ;;
3250             --all)
3251                 ret=0
3252                 echo -n "${testname}: $TEST_DESCRIPTION "
3253                 # Do not use a subshell, otherwise cleanup variables (LOOPDEV) will be lost
3254                 # and loop devices will leak
3255                 test_setup </dev/null >"$TESTLOG" 2>&1 || ret=$?
3256                 if [ $ret -eq 0 ]; then
3257                     test_setup_cleanup </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3258                 fi
3259                 if [ $ret -eq 0 ]; then
3260                     test_run "$TESTID" </dev/null >>"$TESTLOG" 2>&1 || ret=$?
3261                 fi
3262                 test_cleanup
3263                 if [ $ret -eq 0 ]; then
3264                     # $TESTLOG is in $STATEDIR, so clean it up only on success
3265                     [[ -n "$STATEDIR" ]] && rm -vfr "$STATEDIR"
3266                     echo "[OK]"
3267                 else
3268                     echo "[FAILED]"
3269                     echo "see $TESTLOG"
3270                 fi
3271                 exit $ret
3272                 ;;
3273             *)
3274                 break
3275                 ;;
3276         esac
3277         shift
3278     done
3279 }