From: Daan De Meyer <daan.j.demeyer@gmail.com>
Date: Fri, 19 Apr 2024 08:54:38 +0000 (+0200)
Subject: mkosi: Run integration tests in CI
X-Git-Tag: v256-rc1~52^2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F32352%2Fhead;p=thirdparty%2Fsystemd.git

mkosi: Run integration tests in CI

We do the image build and run the tests in a btrfs loopback so we
can make use of btrfs subvolumes and COW to keep the disk space
requirements to a minimum and speed up the ephemeral copies we make
of the image to run the tests.

We also switch to building debug packages and publishing the built
packages as artifacts.
---

diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml
index aa057161812..7e6e64900da 100644
--- a/.github/workflows/mkosi.yml
+++ b/.github/workflows/mkosi.yml
@@ -69,9 +69,6 @@ jobs:
           - distro: centos
             release: "9"
 
-    env:
-      SYSTEMD_LOG_LEVEL: debug
-
     steps:
     - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
     - uses: systemd/mkosi@8cbde8a4ed20a078ad5c70fe38c0dd2294a68bb1
@@ -85,6 +82,18 @@ jobs:
         sudo mv /opt/hostedtoolcache /opt/hostedtoolcache.trash
         sudo systemd-run rm -rf /usr/local.trash /opt/hostedtoolcache.trash
 
+    - name: Btrfs
+      run: |
+        truncate --size=100G btrfs.raw
+        mkfs.btrfs btrfs.raw
+        sudo mkdir /mnt/mkosi
+        LOOP="$(sudo losetup --find --show --direct-io=on btrfs.raw)"
+        sudo mount "$LOOP" /mnt/mkosi --options compress=zstd:1,user_subvol_rm_allowed,noatime,discard=async,space_cache=v2
+        sudo chown "$(id -u):$(id -g)" /mnt/mkosi
+        mkdir /mnt/mkosi/tmp
+        echo "TMPDIR=/mnt/mkosi/tmp" >>"$GITHUB_ENV"
+        ln -s /mnt/mkosi/build build
+
     - name: Configure
       run: |
         tee mkosi.local.conf <<EOF
@@ -95,6 +104,19 @@ jobs:
         [Output]
         # Build a disk image in CI as this logic is much more prone to breakage.
         Format=disk
+        UseSubvolumes=yes
+
+        WorkspaceDirectory=$TMPDIR
+        PackageCacheDirectory=$TMPDIR/cache
+
+        [Content]
+        Environment=
+                # mkfs.erofs is extremely noisy when not connected to a tty.
+                SYSTEMD_REPART_MKFS_OPTIONS_EROFS="--quiet"
+                # Build debuginfo packages since we'll be publishing the packages as artifacts.
+                WITH_DEBUG=1
+                # Enabling optimizations significantly speeds up integration tests.
+                OPTIMIZATION=g
 
         [Host]
         ToolsTree=default
@@ -103,47 +125,82 @@ jobs:
         QemuKvm=yes
         # TODO: Drop once https://bugs.launchpad.net/ubuntu/+source/linux/+bug/2038777 is fixed in Github Actions
         QemuFirmware=uefi
-        Ephemeral=yes
         EOF
 
         # These should override the options from mkosi.conf so we put them in a dropin that's ordered later
         # instead.
         tee mkosi.conf.d/99-ci.conf <<EOF
         [Host]
-        KernelCommandLineExtra=systemd.unit=mkosi-check-and-shutdown.service
-                               systemd.log_level=debug
-                               systemd.journald.max_level_console=debug
-                               # udev's debug log output is very verbose, so up it to info in CI.
-                               udev.log_level=info
-                               # Root device can take a long time to appear, so let's bump the timeout.
-                               systemd.default_device_timeout_sec=180
+        KernelCommandLineExtra=
+                # udev's debug log output is very verbose, so up it to info in CI.
+                udev.log_level=info
+                # Root device can take a long time to appear, so let's bump the timeout.
+                systemd.default_device_timeout_sec=180
         EOF
 
-        # The emergency shell is not useful in the CI, as it just blocks for a long time before the job
-        # eventually times out. Override it to just shutdown immediately.
-        mkdir -p mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/
-        mkdir -p mkosi.images/system/mkosi.extra/usr/lib/systemd/system/emergency.service.d/
-        tee mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf <<EOF
-        [Unit]
-        FailureAction=exit
-        [Service]
-        ExecStartPre=
-        ExecStart=
-        ExecStart=false
-        EOF
-        cp mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf mkosi.images/system/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf
-
     - name: Generate secure boot key
       run: mkosi --debug genkey
 
     - name: Show image summary
       run: mkosi summary
 
-    - name: Build
-      run: mkosi --debug
-
-    - name: Boot systemd-nspawn
-      run: test "$(sudo mkosi --debug boot 1>&2; echo $?)" -eq 123
-
-    - name: Boot QEMU
-      run: timeout -k 30 10m test "$(mkosi --debug qemu 1>&2; echo $?)" -eq 123
+    - name: Install build dependencies
+      run: |
+        sudo apt-get install \
+            meson \
+            gperf \
+            libfdisk-dev \
+            libtss2-dev \
+            libblkid-dev \
+            libmicrohttpd-dev \
+            libcap-dev \
+            libcurl4-openssl-dev \
+            libcryptsetup-dev \
+            erofs-utils \
+            dosfstools \
+            python3-pefile \
+            sbsigntool \
+            mtools
+
+    - name: Configure meson
+      run: |
+        meson setup build \
+            --buildtype=debugoptimized \
+            -Dintegration-tests=true \
+            -Dremote=enabled \
+            -Dopenssl=enabled \
+            -Dblkid=enabled \
+            -Dtpm2=enabled \
+            -Dlibcryptsetup=enabled \
+            -Dlibcurl=enabled \
+            -Drepart=enabled \
+            -Dfirstboot=true \
+            -Dsysusers=true \
+            -Dtmpfiles=true \
+            -Dhwdb=true \
+            -Dvmspawn=enabled
+
+    - name: Build image
+      run: meson compile -C build mkosi
+
+    - name: Run integration tests
+      run: meson test -C build --no-rebuild --suite integration-tests --print-errorlogs --no-stdsplit
+
+    - name: Archive failed test journals
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-failed-test-journals
+        path: |
+          build/test/journal/*.journal
+
+    - name: Archive packages
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-packages
+        path: |
+          build/mkosi.output/*.rpm
+          build/mkosi.output/*.deb
+          build/mkosi.output/*.ddeb
+          build/mkosi.output/*.tar.pkg
diff --git a/mkosi.images/system/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh b/mkosi.images/system/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
deleted file mode 100755
index d2800a04a9f..00000000000
--- a/mkosi.images/system/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash -eux
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-systemctl --failed --no-legend | tee /failed-services
-
-# Check that secure boot keys were properly enrolled.
-if ! systemd-detect-virt --container && \
-   cmp /sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c <(printf '\6\0\0\0\1')
-then
-    cmp /sys/firmware/efi/efivars/SetupMode-8be4df61-93ca-11d2-aa0d-00e098032b8c <(printf '\6\0\0\0\0')
-
-    if command -v sbsign &>/dev/null; then
-        cat /proc/cmdline
-        grep -q this_should_be_here /proc/cmdline
-        (! grep -q this_should_not_be_here /proc/cmdline)
-    fi
-fi
-
-# Exit with non-zero EC if the /failed-services file is not empty (we have -e set)
-[[ ! -s /failed-services ]]
diff --git a/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/mkosi-check-and-shutdown.service b/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/mkosi-check-and-shutdown.service
deleted file mode 100644
index 7942cbfa775..00000000000
--- a/mkosi.images/system/mkosi.extra/usr/lib/systemd/system/mkosi-check-and-shutdown.service
+++ /dev/null
@@ -1,15 +0,0 @@
-# SPDX-License-Identifier: LGPL-2.1-or-later
-[Unit]
-Description=Check if any service failed and then shutdown the machine
-After=multi-user.target network-online.target
-Requires=multi-user.target
-Wants=systemd-resolved.service systemd-networkd.service network-online.target
-SuccessAction=exit
-FailureAction=exit
-# On success, exit with 123 so that we can check that we receive the actual exit code from the script on the
-# host.
-SuccessActionExitStatus=123
-
-[Service]
-Type=oneshot
-ExecStart=/usr/lib/systemd/mkosi-check-and-shutdown.sh