feat(systemd-import): introducing the systemd-import module

This new module takes advantage of the features provided by
systemd-import-generator(8), systemd-importd.service(8) and
systemd-loop@.service(8), especifically the `rd.systemd.pull=` kernel command
line option, to download a disk image (tar/raw) into memory, optionally validate
its checksum or signature, and directly boot into it.

It allows to:
- Download a tar disk image into /run/machines and bind mount it into /sysroot
(via `root=bind:...`).
- Download a raw image into memory and attach it to a loopback block device, so
we can point `root=` to a known label or to the proper `/dev/disk/by-loop-ref/`
device.

Notes:
- It needs enough RAM to save and unpack/decompress the image.
- The image can be compressed with xz, gzip, bzip2, zstd.
- It supports btrfs, erofs, ext4, f2fs, squashfs, vfat, or xfs filesystems.

diff --git a/.github/labeler.yml b/.github/labeler.yml
index 65ba4b4f20781882a3d5282920e44d1b9af2262c..ae18ae121882126e944dfcdc66c640210801bd31 100644 (file)
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -111,6 +111,10 @@ systemd-hostnamed:
     - changed-files:
           - any-glob-to-any-file: 'modules.d/[0-9][0-9]systemd-hostnamed/*'
 
+systemd-import:
+    - changed-files:
+          - any-glob-to-any-file: 'modules.d/[0-9][0-9]systemd-import/*'
+
 systemd-initrd:
     - changed-files:
           - any-glob-to-any-file: 'modules.d/[0-9][0-9]systemd-initrd/*'

diff --git a/.github/workflows/daily-systemd.yml b/.github/workflows/daily-systemd.yml
index 6f5b5726d01a957959636783f4be5d38df89e22f..05de65891dc8199c456488a930dacc016f53aac5 100644 (file)
--- a/.github/workflows/daily-systemd.yml
+++ b/.github/workflows/daily-systemd.yml
@@ -46,6 +46,7 @@ jobs:
                     - "42"
                     - "43"
                     - "44"
+                    - "45"
                 exclude:
                     - container: arch:latest
                       architecture: {runner: 'ubuntu-24.04-arm', tag: 'arm'}
@@ -74,6 +75,21 @@ jobs:
                     # https://github.com/dracut-ng/dracut-ng/issues/1988
                     - container: debian:sid
                       architecture: {runner: 'ubuntu-24.04-arm', tag: 'arm'}
+                    # TEST-45-SYSTEMD-IMPORT requires systemd >= v258
+                    - container: azurelinux:3.0
+                      test: "45"
+                    # TEST-45-SYSTEMD-IMPORT requires systemd >= v258
+                    - container: centos:latest
+                      test: "45"
+                    # TEST-45-SYSTEMD-IMPORT requires systemd >= v258
+                    - container: debian:latest
+                      test: "45"
+                    # TEST-45-SYSTEMD-IMPORT requires systemd >= v258
+                    - container: opensuse:latest
+                      test: "45"
+                    # TEST-45-SYSTEMD-IMPORT requires systemd >= v258
+                    - container: ubuntu:rolling
+                      test: "45"
         container:
             image: ghcr.io/dracut-ng/${{ matrix.container }}
             options: '--device=/dev/kvm --privileged'

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 70987dbac600077495e905a5aae7cd23dd690617..5eacac0aa358739e14095e20544cac8f5892703e 100644 (file)
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -132,10 +132,14 @@ jobs:
                     - "42"
                     - "43"
                     - "44"
+                    - "45"
                 exclude:
                     # https://github.com/dracut-ng/dracut-ng/issues/1677
                     - container: arch:latest
                       test: "41"
+                    # TEST-45-SYSTEMD-IMPORT requires systemd >= v258
+                    - container: opensuse:latest
+                      test: "45"
         container:
             image: ghcr.io/dracut-ng/${{ matrix.container }}
             options: '--device=/dev/kvm --privileged'

diff --git a/doc_site/modules/ROOT/pages/modules/systemd.adoc b/doc_site/modules/ROOT/pages/modules/systemd.adoc
index 2c8423d3510e35cc452a1d73452291c976636201..e31fd20617519f44b4c0c3cb7b7882ea5a6da923 100644 (file)
--- a/doc_site/modules/ROOT/pages/modules/systemd.adoc
+++ b/doc_site/modules/ROOT/pages/modules/systemd.adoc
@@ -80,6 +80,9 @@ These modules would require including a version of systemd into initramfs.
 | systemd-hostnamed
 | https://www.freedesktop.org/software/systemd/man/systemd-hostnamed.html[systemd-hostnamed]
 
+| systemd-import
+| https://www.freedesktop.org/software/systemd/man/devel/systemd-import-generator.html[systemd-import]
+
 | systemd-initrd
 | https://systemd.io/INITRD_INTERFACE/[INITRD_INTERFACE]
 

diff --git a/modules.d/10systemd/module-setup.sh b/modules.d/10systemd/module-setup.sh
index 509dd68669144b1669dd7159cc0e3df71e71506e..1352688c04c3eaf4cd6ee34ddbbb6c0c7b7d2dae 100755 (executable)
--- a/modules.d/10systemd/module-setup.sh
+++ b/modules.d/10systemd/module-setup.sh
@@ -92,7 +92,7 @@ install() {
         "$systemdsystemunitdir"/slices.target \
         "$systemdsystemunitdir"/system.slice \
         "$systemdsystemunitdir"/-.slice \
-        systemctl \
+        systemctl varlinkctl \
         echo swapoff \
         chmod \
         mount umount reboot poweroff \

diff --git a/modules.d/45systemd-import/module-setup.sh b/modules.d/45systemd-import/module-setup.sh
new file mode 100755 (executable)
index 0000000..26271e8
--- /dev/null
+++ b/modules.d/45systemd-import/module-setup.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+check() {
+    require_binaries \
+        "$systemdutildir"/systemd-importd \
+        || return 1
+
+    return 255
+}
+
+# due to the dependencies below, this dracut module needs to be ordered later
+# than the network dracut modules
+depends() {
+    echo systemd network
+    return 0
+}
+
+config() {
+    add_dlopen_features+=" libsystemd-shared-*.so:archive,lz4,lzma,zstd "
+}
+
+installkernel() {
+    hostonly='' instmods btrfs erofs ext4 f2fs loop squashfs vfat xfs
+
+    # xfs/btrfs/ext4 need crc32c, f2fs needs crc32,
+    # vfat needs charsets for codepage= and iocharset=
+    hostonly='' instmods crc32c crc32 "=fs/nls"
+}
+
+install() {
+    local _systemd_version
+    _systemd_version=$(udevadm --version 2> /dev/null)
+
+    inst_hook cmdline 10 "$moddir/parse-systemd-import.sh"
+
+    inst_multiple -o \
+        "$dbussystem"/org.freedesktop.import1.conf \
+        "$dbussystemservices"/org.freedesktop.import1.service \
+        "$systemdutildir"/import-pubring.pgp \
+        "$systemdutildir"/systemd-import \
+        "$systemdutildir"/systemd-import-fs \
+        "$systemdutildir"/systemd-importd \
+        "$systemdutildir"/systemd-pull \
+        "$systemdutildir"/system-generators/systemd-import-generator \
+        "$systemdsystemunitdir"/systemd-importd.service \
+        "$systemdsystemunitdir"/systemd-importd.socket \
+        "$systemdsystemunitdir"/systemd-loop@.service \
+        "$systemdsystemunitdir"/imports.target \
+        "$systemdsystemunitdir"/imports-pre.target \
+        "$systemdsystemunitdir"/dbus-org.freedesktop.import1.service \
+        "$systemdsystemunitdir"/sockets.target.wants/systemd-importd.socket \
+        "$systemdsystemunitdir"/sysinit.target.wants/imports.target \
+        gpg systemd-dissect
+
+    # systemd < v259: requires the tar binary
+    # See: https://github.com/systemd/systemd/commit/a7c8f92d1f937113a279adbe62399f6f0773473f
+    if ((_systemd_version < 259)); then
+        inst_multiple -o \
+            tar
+    fi
+
+    if [[ $hostonly ]]; then
+        inst_multiple -H -o \
+            "$systemdutilconfdir"/import-pubring.pgp \
+            "$systemdsystemconfdir"/systemd-importd.service \
+            "$systemdsystemconfdir/systemd-importd.service.d/*.conf" \
+            "$systemdsystemconfdir"/systemd-importd.socket \
+            "$systemdsystemconfdir/systemd-importd.socket.d/*.conf" \
+            "$systemdsystemconfdir"/systemd-loop@.service \
+            "$systemdsystemconfdir/systemd-loop@.service.d/*.conf" \
+            "$systemdsystemconfdir"/imports.target \
+            "$systemdsystemconfdir/imports.target.wants/*.target" \
+            "$systemdsystemconfdir"/imports-pre.target \
+            "$systemdsystemconfdir/imports-pre.target.wants/*.target"
+    fi
+
+    # libcurl and libopenssl are not loaded via dlopen
+    _arch=${DRACUT_ARCH:-$(uname -m)}
+    inst_libdir_file \
+        {"tls/$_arch/",tls/,"$_arch/",}"libcurl.so*" \
+        {"tls/$_arch/",tls/,"$_arch/",}"libssl.so*"
+
+    if [[ ! $USE_SYSTEMD_DLOPEN_DEPS ]]; then
+        inst_libdir_file \
+            {"tls/$_arch/",tls/,"$_arch/",}"libarchive.so*" \
+            {"tls/$_arch/",tls/,"$_arch/",}"liblz4.so*" \
+            {"tls/$_arch/",tls/,"$_arch/",}"liblzma.so*" \
+            {"tls/$_arch/",tls/,"$_arch/",}"libzstd.so*"
+    fi
+}

diff --git a/modules.d/45systemd-import/parse-systemd-import.sh b/modules.d/45systemd-import/parse-systemd-import.sh
new file mode 100755 (executable)
index 0000000..e1afeed
--- /dev/null
+++ b/modules.d/45systemd-import/parse-systemd-import.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+command -v getarg > /dev/null || . /lib/dracut-lib.sh
+
+if getarg rd.systemd.pull= > /dev/null; then
+    echo "rd.neednet=1" > /etc/cmdline.d/01-systemd-import.conf
+    if ! getarg "ip="; then
+        echo "ip=dhcp" >> /etc/cmdline.d/01-systemd-import.conf
+    fi
+fi

diff --git a/modules.d/77dracut-systemd/dracut-cmdline.sh b/modules.d/77dracut-systemd/dracut-cmdline.sh
index c0f42e920018f009a2074073c86efb17e33134cb..cc23fb1131a3b3b9f28412730b511d979d7ef22a 100755 (executable)
--- a/modules.d/77dracut-systemd/dracut-cmdline.sh
+++ b/modules.d/77dracut-systemd/dracut-cmdline.sh
@@ -59,7 +59,7 @@ case "${root#block:}${root_unset}" in
         root="block:${root#block:}"
         rootok=1
         ;;
-    UNSET | gpt-auto | gpt-auto-force | dissect | dissect-force | fstab | tmpfs | off)
+    UNSET | gpt-auto | gpt-auto-force | dissect | dissect-force | fstab | tmpfs | bind:* | off)
         # systemd's gpt-auto-generator/fstab-generator handles this case.
         rootok=1
         ;;

diff --git a/test/TEST-45-SYSTEMD-IMPORT/Makefile b/test/TEST-45-SYSTEMD-IMPORT/Makefile
new file mode 100644 (file)
index 0000000..2dcab81
--- /dev/null
+++ b/test/TEST-45-SYSTEMD-IMPORT/Makefile
@@ -0,0 +1 @@
+-include ../Makefile.testdir

diff --git a/test/TEST-45-SYSTEMD-IMPORT/test.sh b/test/TEST-45-SYSTEMD-IMPORT/test.sh
new file mode 100755 (executable)
index 0000000..5c3e3b1
--- /dev/null
+++ b/test/TEST-45-SYSTEMD-IMPORT/test.sh
@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+set -eu
+
+# shellcheck disable=SC2034
+TEST_DESCRIPTION="download and import disk images at boot with systemd-import"
+
+# Uncomment these to debug failures
+#DEBUGFAIL="systemd.show_status=1 systemd.log_level=debug"
+
+# Verify checksum by default
+IMPORT_VERIFY="checksum"
+
+test_check() {
+    local binary
+
+    for binary in /usr/lib/systemd/systemd-importd systemd-dissect tar zstd; do
+        if ! type -p "$binary" &> /dev/null; then
+            echo "Test needs $binary... Skipping"
+            return 1
+        fi
+    done
+}
+
+client_run() {
+    local test_name="$1"
+    local append="$2"
+
+    client_test_start "$test_name"
+
+    "$testdir"/run-qemu \
+        -device "virtio-net-pci,netdev=lan0" \
+        -netdev "user,id=lan0,net=10.0.2.0/24,dhcpstart=10.0.2.15" \
+        -append "$append $TEST_KERNEL_CMDLINE" \
+        -initrd "$TESTDIR/initramfs.testing"
+    check_qemu_log
+
+    client_test_end
+}
+
+test_run() {
+    local port root_dir image_name
+
+    port=$(start_webserver)
+
+    root_dir="root"
+    client_run "Download tar disk image into /run/machines/$root_dir, verify $IMPORT_VERIFY and bind mount it into /sysroot" \
+        "rd.systemd.pull=tar,machine,verify=$IMPORT_VERIFY:$root_dir:http://10.0.2.2:$port/root.tar.zst root=bind:/run/machines/$root_dir"
+
+    image_name="image"
+    client_run "Download ext4 raw image into memory, verify $IMPORT_VERIFY and attach it to a loopback block device" \
+        "rd.systemd.pull=raw,machine,verify=$IMPORT_VERIFY,blockdev:$image_name:http://10.0.2.2:$port/root_ext4.img root=/dev/disk/by-loop-ref/$image_name.raw"
+
+    if command -v mksquashfs &> /dev/null; then
+        client_run "Download squashfs image into memory, verify $IMPORT_VERIFY and attach it to a loopback block device" \
+            "rd.systemd.pull=raw,machine,verify=$IMPORT_VERIFY,blockdev:$image_name:http://10.0.2.2:$port/root_squashfs.img root=/dev/disk/by-loop-ref/$image_name.raw"
+    fi
+
+    if command -v mkfs.erofs &> /dev/null \
+        && ! grep -q -r -w "blacklist erofs" /{etc,usr/lib}/modprobe.d; then
+        client_run "Download erofs image into memory, verify $IMPORT_VERIFY and attach it to a loopback block device" \
+            "rd.systemd.pull=raw,machine,verify=$IMPORT_VERIFY,blockdev:$image_name:http://10.0.2.2:$port/root_erofs.img root=/dev/disk/by-loop-ref/$image_name.raw"
+    fi
+}
+
+test_setup() {
+    local image images=() local_pubring=() network_handler
+
+    # Create plain root filesystem
+    build_client_rootfs "$TESTDIR/rootfs"
+
+    # Create a compressed tarball with the plain rootfs
+    tar -C "$TESTDIR/rootfs" --zstd -cf "$TESTDIR/root.tar.zst" .
+    images+=("root.tar.zst")
+
+    # Create an ext4 image with the rootfs
+    build_ext4_image "$TESTDIR/rootfs" "$TESTDIR/root_ext4.img" dracut
+    images+=("root_ext4.img")
+
+    # If mksquashfs is available, create a squashfs image with the rootfs
+    if command -v mksquashfs &> /dev/null; then
+        mksquashfs "$TESTDIR/rootfs" "$TESTDIR/root_squashfs.img" -quiet -no-progress
+        images+=("root_squashfs.img")
+    fi
+
+    # erofs is supported, but can be explicitly blacklisted by the OS
+    # (e.g., in openSUSE distributions)
+    if command -v mkfs.erofs &> /dev/null \
+        && ! grep -q -r -w "blacklist erofs" /{etc,usr/lib}/modprobe.d; then
+        mkfs.erofs "$TESTDIR/root_erofs.img" "$TESTDIR/rootfs"
+        images+=("root_erofs.img")
+    fi
+
+    # If GnuPG is available, generate a key and import it into a keyring file
+    # that will be checked by systemd-import, so it can verify the signature of
+    # the checksums of each image
+    if command -v gpg &> /dev/null && command -v gpg-agent &> /dev/null; then
+        mkdir -m 700 "$TESTDIR/gpg"
+        gpg --homedir "$TESTDIR/gpg" --batch --quick-generate-key --passphrase "" "dracut" ed25519 default 0
+        gpg --homedir "$TESTDIR/gpg" --output "$TESTDIR/dracut.asc" --export -a "dracut"
+        gpg --homedir "$TESTDIR/gpg" --no-default-keyring --keyring "$TESTDIR/import-pubring.pgp" --import "$TESTDIR/dracut.asc"
+        local_pubring=("-i" "$TESTDIR/import-pubring.pgp" "/etc/systemd/import-pubring.pgp")
+        IMPORT_VERIFY="signature"
+    fi
+
+    # Save sha256 checksums, and sign them if GnuPG is available
+    pushd "$TESTDIR"
+    for image in "${images[@]}"; do
+        sha256sum "$image" > "$image.sha256"
+        if [[ $IMPORT_VERIFY == "signature" ]]; then
+            gpg --homedir "$TESTDIR/gpg" --detach-sig --sign -u "dracut" -a "$image.sha256"
+        fi
+    done
+    popd
+
+    # If systemd-networkd is installed, give preference to it, because
+    # network-manager uses dbus, and depending on the specific implementation
+    # of the broker (e.g. Ubuntu), it can lead to ordering cycles.
+    network_handler=""
+    if [[ -x /usr/lib/systemd/systemd-networkd ]]; then
+        network_handler="systemd-networkd"
+    fi
+    test_dracut \
+        --no-hostonly-cmdline \
+        "${local_pubring[@]}" \
+        -a "systemd-import $network_handler"
+}
+
+test_cleanup() {
+    stop_webserver
+}
+
+# shellcheck disable=SC1090
+. "$testdir"/test-functions