Add esp output format

This is largely the same as the uki output format, but additionally
we wrap the uki in a disk image with just an ESP.

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6dedc7a7195a2fb7e7787beee01702a9a8103230..6edc3e8457a627a817bef6174f421e56c5d3093c 100644 (file)
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -134,9 +134,12 @@ jobs:
           - cpio
           - disk
           - uki
+          - esp
         exclude:
           - distro: rhel-ubi
             format: uki
+          - distro: rhel-ubi
+            format: esp
 
     steps:
     - uses: actions/checkout@v3
@@ -182,7 +185,7 @@ jobs:
       run: sudo mkosi --debug boot
 
     - name: Boot ${{ matrix.distro }}/${{ matrix.format }} QEMU
-      if: matrix.distro != 'rhel-ubi' && (matrix.format == 'disk' || matrix.format == 'uki' || matrix.format == 'cpio')
+      if: matrix.distro != 'rhel-ubi' && matrix.format != 'directory' && matrix.format != 'tar'
       run: timeout -k 30 10m mkosi --debug qemu
 
     - name: Boot ${{ matrix.distro }}/${{ matrix.format }} BIOS

diff --git a/mkosi/__init__.py b/mkosi/__init__.py
index c8833aa4b90996453c26cc80a14de75b79d4359e..5f3b17b05c1f185c21b4bae903842f15d70d3891 100644 (file)
--- a/mkosi/__init__.py
+++ b/mkosi/__init__.py
@@ -1319,7 +1319,7 @@ def want_uki(config: MkosiConfig) -> bool:
     # Note that this returns True also in the case where autodetection might later
     # cause the UKI not to be installed after the file system has been populated.
 
-    if config.output_format == OutputFormat.uki:
+    if config.output_format in (OutputFormat.uki, OutputFormat.esp):
         return True
 
     if config.bootable == ConfigFeature.disabled:
@@ -1344,7 +1344,7 @@ def install_uki(state: MkosiState, partitions: Sequence[Partition]) -> None:
     # benefit that they can be signed like normal EFI binaries, and can encode everything necessary to boot a
     # specific root device, including the root hash.
 
-    if not want_uki(state.config) or state.config.output_format == OutputFormat.uki:
+    if not want_uki(state.config) or state.config.output_format in (OutputFormat.uki, OutputFormat.esp):
         return
 
     arch = state.config.architecture.to_efi()
@@ -1682,7 +1682,7 @@ def check_tools(args: MkosiArgs, config: MkosiConfig) -> None:
             hint="Bootable=no can be used to create a non-bootable image",
         )
 
-    if config.output_format == OutputFormat.disk:
+    if config.output_format in (OutputFormat.disk, OutputFormat.esp):
         check_systemd_tool("systemd-repart", reason="build disk images")
 
     if args.verb == Verb.boot:
@@ -1956,10 +1956,14 @@ def reuse_cache(state: MkosiState) -> bool:
     return True
 
 
-def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: bool = False) -> list[Partition]:
-    if not state.config.output_format == OutputFormat.disk:
-        return []
-
+def make_image(
+    state: MkosiState,
+    msg: str,
+    skip: Sequence[str] = [],
+    split: bool = False,
+    root: Optional[Path] = None,
+    definitions: Sequence[Path] = [],
+) -> list[Partition]:
     cmdline: list[PathString] = [
         "systemd-repart",
         "--empty=allow",
@@ -1968,11 +1972,12 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo
         "--json=pretty",
         "--no-pager",
         "--offline=yes",
-        "--root", state.root,
         "--seed", str(state.config.seed) if state.config.seed else "random",
         state.staging / state.config.output_with_format,
     ]
 
+    if root:
+        cmdline += ["--root", root]
     if not state.config.architecture.is_native():
         cmdline += ["--architecture", str(state.config.architecture)]
     if not (state.staging / state.config.output_with_format).exists():
@@ -1990,19 +1995,52 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo
     if state.config.sector_size:
         cmdline += ["--sector-size", state.config.sector_size]
 
-    if state.config.repart_dirs:
-        for d in state.config.repart_dirs:
+    if definitions:
+        for d in definitions:
             cmdline += ["--definitions", d]
 
         # Subvolumes= only works with --offline=no.
-        grep = run(["grep", "--recursive", "--include=*.conf", "Subvolumes=", *state.config.repart_dirs],
+        grep = run(["grep", "--recursive", "--include=*.conf", "Subvolumes=", *definitions],
                    stdout=subprocess.DEVNULL, check=False)
         if grep.returncode == 0:
             cmdline += ["--offline=no"]
+
+    env = {
+        option: value
+        for option, value in state.config.environment.items()
+        if option.startswith("SYSTEMD_REPART_MKFS_OPTIONS_") or option == "SOURCE_DATE_EPOCH"
+    }
+
+    with complete_step(msg):
+        output = json.loads(run(cmdline, stdout=subprocess.PIPE, env=env).stdout)
+
+    logging.debug(json.dumps(output, indent=4))
+
+    partitions = [Partition.from_dict(d) for d in output]
+
+    if split:
+        for p in partitions:
+            if p.split_path:
+                maybe_compress(state.config, state.config.compress_output, p.split_path)
+
+    return partitions
+
+
+def make_disk(
+    state: MkosiState,
+    msg: str,
+    skip: Sequence[str] = [],
+    split: bool = False,
+) -> list[Partition]:
+    if state.config.output_format != OutputFormat.disk:
+        return []
+
+    if state.config.repart_dirs:
+        definitions = state.config.repart_dirs
     else:
-        definitions = state.workspace / "repart-definitions"
-        if not definitions.exists():
-            definitions.mkdir()
+        defaults = state.workspace / "repart-definitions"
+        if not defaults.exists():
+            defaults.mkdir()
             if (arch := state.config.architecture.to_efi()):
                 bootloader = state.root / f"efi/EFI/BOOT/BOOT{arch.upper()}.EFI"
             else:
@@ -2012,7 +2050,7 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo
             bios = (state.config.bootable != ConfigFeature.disabled and want_grub_bios(state))
 
             if bios:
-                (definitions / "05-bios.conf").write_text(
+                (defaults / "05-bios.conf").write_text(
                     textwrap.dedent(
                         f"""\
                         [Partition]
@@ -2032,7 +2070,7 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo
                 # Even if we're doing BIOS, let's still use the ESP to store the kernels, initrds and grub
                 # modules. We cant use UKIs so we have to put each kernel and initrd on the ESP twice, so
                 # let's make the ESP twice as big in that case.
-                (definitions / "00-esp.conf").write_text(
+                (defaults / "00-esp.conf").write_text(
                     textwrap.dedent(
                         f"""\
                         [Partition]
@@ -2045,7 +2083,7 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo
                     )
                 )
 
-            (definitions / "10-root.conf").write_text(
+            (defaults / "10-root.conf").write_text(
                 textwrap.dedent(
                     f"""\
                     [Partition]
@@ -2057,27 +2095,38 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo
                 )
             )
 
-        cmdline += ["--definitions", definitions]
+        definitions = [defaults]
 
-    env = {
-        option: value
-        for option, value in state.config.environment.items()
-        if option.startswith("SYSTEMD_REPART_MKFS_OPTIONS_") or option == "SOURCE_DATE_EPOCH"
-    }
+    return make_image(state, msg=msg, skip=skip, split=split, root=state.root, definitions=definitions)
 
-    with complete_step(msg):
-        output = json.loads(run(cmdline, stdout=subprocess.PIPE, env=env).stdout)
 
-    logging.debug(json.dumps(output, indent=4))
+def make_esp(state: MkosiState, uki: Path) -> list[Partition]:
+    if not (arch := state.config.architecture.to_efi()):
+        die(f"Architecture {state.config.architecture} does not support UEFI")
 
-    partitions = [Partition.from_dict(d) for d in output]
+    definitions = state.workspace / "esp-definitions"
+    definitions.mkdir(exist_ok=True)
 
-    if split:
-        for p in partitions:
-            if p.split_path:
-                maybe_compress(state.config, state.config.compress_output, p.split_path)
+    # Use a minimum of 512MB because otherwise the generated FAT filesystem will have too few clusters to be considered
+    # a FAT32 filesystem by OVMF which will refuse to boot from it. Always reserve 10MB for filesystem metadata.
+    size = max(uki.stat().st_size, 502 * 1024**2) + 10 * 1024**2
 
-    return partitions
+    # TODO: Remove the extra 4096 for the max size once https://github.com/systemd/systemd/pull/29954 is in a stable
+    # release.
+    (definitions / "00-esp.conf").write_text(
+        textwrap.dedent(
+            f"""\
+            [Partition]
+            Type=esp
+            Format=vfat
+            CopyFiles={uki}:/EFI/BOOT/BOOT{arch.upper()}.EFI
+            SizeMinBytes={size}
+            SizeMaxBytes={size + 4096}
+            """
+        )
+    )
+
+    return make_image(state, msg="Generating ESP image", definitions=[definitions])
 
 
 def finalize_staging(state: MkosiState) -> None:
@@ -2192,17 +2241,17 @@ def build_image(args: MkosiArgs, config: MkosiConfig) -> None:
             run_finalize_scripts(state)
 
         normalize_mtime(state.root, state.config.source_date_epoch)
-        partitions = make_image(state, skip=("esp", "xbootldr"), msg="Generating disk image")
+        partitions = make_disk(state, skip=("esp", "xbootldr"), msg="Generating disk image")
         install_uki(state, partitions)
         prepare_grub_efi(state)
         prepare_grub_bios(state, partitions)
         normalize_mtime(state.root, state.config.source_date_epoch, directory=Path("boot"))
         normalize_mtime(state.root, state.config.source_date_epoch, directory=Path("efi"))
-        partitions = make_image(state, msg="Formatting ESP/XBOOTLDR partitions")
+        partitions = make_disk(state, msg="Formatting ESP/XBOOTLDR partitions")
         install_grub_bios(state, partitions)
 
         if state.config.split_artifacts:
-            make_image(state, split=True, msg="Extracting partitions")
+            make_disk(state, split=True, msg="Extracting partitions")
 
         copy_vmlinuz(state)
 
@@ -2212,10 +2261,13 @@ def build_image(args: MkosiArgs, config: MkosiConfig) -> None:
             make_cpio(state.root, state.staging / state.config.output_with_format)
         elif state.config.output_format == OutputFormat.uki:
             make_uki(state, state.staging / state.config.output_with_format)
+        elif state.config.output_format == OutputFormat.esp:
+            make_uki(state, state.staging / state.config.output_split_uki)
+            make_esp(state, state.staging / state.config.output_split_uki)
         elif state.config.output_format == OutputFormat.directory:
             state.root.rename(state.staging / state.config.output_with_format)
 
-        if config.output_format != OutputFormat.uki:
+        if config.output_format not in (OutputFormat.uki, OutputFormat.esp):
             maybe_compress(state.config, state.config.compress_output,
                            state.staging / state.config.output_with_format,
                            state.staging / state.config.output_with_compression)

diff --git a/mkosi/config.py b/mkosi/config.py
index 93be0eb0c7bee071d7128ffeef7edfc159929b52..ae78396fbc34e55af9c4dd55fd4c9dd4f9b633b9 100644 (file)
--- a/mkosi/config.py
+++ b/mkosi/config.py
@@ -104,11 +104,13 @@ class OutputFormat(StrEnum):
     cpio      = enum.auto()
     disk      = enum.auto()
     uki       = enum.auto()
+    esp       = enum.auto()
     none      = enum.auto()
 
     def extension(self) -> str:
         return {
             OutputFormat.disk: ".raw",
+            OutputFormat.esp:  ".raw",
             OutputFormat.cpio: ".cpio",
             OutputFormat.tar:  ".tar",
             OutputFormat.uki:  ".efi",
@@ -302,7 +304,7 @@ def config_parse_source_date_epoch(value: Optional[str], old: Optional[int]) ->
 
 
 def config_default_compression(namespace: argparse.Namespace) -> Compression:
-    if namespace.output_format in (OutputFormat.cpio, OutputFormat.uki):
+    if namespace.output_format in (OutputFormat.cpio, OutputFormat.uki, OutputFormat.esp):
         if namespace.distribution.is_centos_variant() and int(namespace.release) <= 8:
             return Compression.xz
         else:
@@ -952,7 +954,7 @@ class MkosiConfig:
     def output_with_compression(self) -> str:
         output = self.output_with_format
 
-        if self.compress_output and self.output_format != OutputFormat.uki:
+        if self.compress_output and self.output_format not in (OutputFormat.uki, OutputFormat.esp):
             output += f".{self.compress_output}"
 
         return output
@@ -2894,7 +2896,7 @@ Clean Package Manager Metadata: {yes_no_auto(config.clean_package_metadata)}
                            SSH: {yes_no(config.ssh)}
 """
 
-    if config.output_format == OutputFormat.disk:
+    if config.output_format in (OutputFormat.disk, OutputFormat.uki, OutputFormat.esp):
         summary += f"""\
 
     {bold("VALIDATION")}:

diff --git a/mkosi/qemu.py b/mkosi/qemu.py
index 12d49a57a2c3bd7c047e64bbc84d331ef72596f9..69c64537999f45c92d46b4f14cb60076432b6fac 100644 (file)
--- a/mkosi/qemu.py
+++ b/mkosi/qemu.py
@@ -366,11 +366,17 @@ def qemu_version(config: MkosiConfig) -> GenericVersion:
 
 
 def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[QemuDeviceNode, int]) -> None:
-    if config.output_format not in (OutputFormat.disk, OutputFormat.cpio, OutputFormat.uki, OutputFormat.directory):
+    if config.output_format not in (
+        OutputFormat.disk,
+        OutputFormat.cpio,
+        OutputFormat.uki,
+        OutputFormat.esp,
+        OutputFormat.directory,
+    ):
         die(f"{config.output_format} images cannot be booted in qemu")
 
     if (
-        config.output_format in (OutputFormat.cpio, OutputFormat.uki) and
+        config.output_format in (OutputFormat.cpio, OutputFormat.uki, OutputFormat.esp) and
         config.qemu_firmware not in (QemuFirmware.auto, QemuFirmware.linux, QemuFirmware.uefi)
     ):
         die(f"{config.output_format} images cannot be booted with the '{config.qemu_firmware}' firmware")
@@ -396,7 +402,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[Qemu
     else:
         kernel = None
 
-    if config.output_format == OutputFormat.uki and kernel:
+    if config.output_format in (OutputFormat.uki, OutputFormat.esp) and kernel:
         logging.warning(
             f"Booting UKI output, kernel {kernel} configured with QemuKernel= or passed with -kernel will not be used"
         )
@@ -514,8 +520,8 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[Qemu
                 "-drive", f"file={ovmf_vars.name},if=pflash,format=raw",
             ]
 
-        if config.qemu_cdrom and config.output_format == OutputFormat.disk:
-            # CD-ROM devices have sector size 2048 so we transform the disk image into one with sector size 2048.
+        if config.qemu_cdrom and config.output_format in (OutputFormat.disk, OutputFormat.esp):
+            # CD-ROM devices have sector size 2048 so we transform disk images into ones with sector size 2048.
             src = (config.output_dir_or_cwd() / config.output).resolve()
             fname = src.parent / f"{src.name}-{uuid.uuid4().hex}"
             run(["systemd-repart",
@@ -600,7 +606,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[Qemu
         ):
             cmdline += ["-initrd", config.output_dir_or_cwd() / config.output_split_initrd]
 
-        if config.output_format == OutputFormat.disk:
+        if config.output_format in (OutputFormat.disk, OutputFormat.esp):
             cmdline += ["-drive", f"if=none,id=mkosi,file={fname},format=raw",
                         "-device", "virtio-scsi-pci,id=scsi",
                         "-device", f"scsi-{'cd' if config.qemu_cdrom else 'hd'},drive=mkosi,bootindex=1"]

diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md
index 94718651d6f2d9a32ccac10277b6552433718800..6bfeb37a7bb4f8f7013f41a403e8f20520d426a2 100644 (file)
--- a/mkosi/resources/mkosi.md
+++ b/mkosi/resources/mkosi.md
@@ -557,12 +557,14 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`,
 
 `Format=`, `--format=`, `-t`
 
-: The image format type to generate. One of `directory` (for generating an OS
-  image directly in a local directory), `tar` (similar, but a tarball of the OS
-  image is generated), `cpio` (similar, but a cpio archive is generated),
-  `disk` (a block device OS image with a GPT partition table), `uki` (a unified
-  kernel image with the OS image in the `.initrd` PE section) or `none` (the OS
-  image is solely intended as a build image to produce another artifact).
+: The image format type to generate. One of `directory` (for generating
+  an OS image directly in a local directory), `tar` (similar, but a
+  tarball of the OS image is generated), `cpio` (similar, but a cpio
+  archive is generated), `disk` (a block device OS image with a GPT
+  partition table), `uki` (a unified kernel image with the OS image in
+  the `.initrd` PE section), `esp` (`uki` but wrapped in a disk image
+  with only an ESP partition) or `none` (the OS image is solely intended
+  as a build image to produce another artifact).
 
 : If the `disk` output format is used, the disk image is generated using
   `systemd-repart`. The repart partition definition files to use can be