From 8f26a6da17e6c56c4135ce0e87426539186a0935 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 7 Sep 2023 15:08:41 +0200 Subject: [PATCH] Support direct kernel booting a disk image This requires two things: 1. We need to generate a split initrd again to pass to -initrd 2. We need to synthesize a root= argument as we can't rely on gpt-auto-generator since we're not using EFI. We move the partition and root= specific stuff to a new file partition.py so we can access it from qemu.py as well. We also introduce extract_pe_section() since we now use the logic twice. --- mkosi/__init__.py | 94 ++++++++++++++-------------------------------- mkosi/config.py | 4 ++ mkosi/partition.py | 65 ++++++++++++++++++++++++++++++++ mkosi/qemu.py | 15 +++++++- 4 files changed, 112 insertions(+), 66 deletions(-) create mode 100644 mkosi/partition.py diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 20010d326..fbe56030e 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -19,7 +19,7 @@ import textwrap import uuid from collections.abc import Iterator, Sequence from pathlib import Path -from typing import Any, ContextManager, Mapping, Optional, TextIO, Union +from typing import ContextManager, Optional, TextIO, Union from mkosi.archive import extract_tar, make_cpio, make_tar from mkosi.config import ( @@ -45,6 +45,7 @@ from mkosi.log import ARG_DEBUG, complete_step, die, log_step from mkosi.manifest import Manifest from mkosi.mounts import mount, mount_overlay, mount_passwd, mount_usr from mkosi.pager import page +from mkosi.partition import Partition, finalize_root, finalize_roothash from mkosi.qemu import copy_ephemeral, run_qemu, run_ssh from mkosi.run import become_root, bwrap, chroot_cmd, init_mount_namespace, run from mkosi.state import MkosiState @@ -63,27 +64,6 @@ from mkosi.util import ( from mkosi.versioncomp import GenericVersion -@dataclasses.dataclass(frozen=True) -class Partition: - type: str - uuid: str - partno: Optional[int] - split_path: Optional[Path] - roothash: Optional[str] - - @classmethod - def from_dict(cls, dict: Mapping[str, Any]) -> "Partition": - return cls( - type=dict["type"], - uuid=dict["uuid"], - partno=int(partno) if (partno := dict.get("partno")) else None, - split_path=Path(p) if ((p := dict.get("split_path")) and p != "-") else None, - roothash=dict.get("roothash"), - ) - - GRUB_BOOT_PARTITION_UUID = "21686148-6449-6e6f-744e-656564454649" - - @contextlib.contextmanager def mount_base_trees(state: MkosiState) -> Iterator[None]: if not state.config.base_trees or not state.config.overlay: @@ -708,12 +688,7 @@ def prepare_grub_bios(state: MkosiState, partitions: Sequence[Partition]) -> Non config = prepare_grub_config(state) assert config - root = finalize_roothash(partitions) - if not root: - root = next((f"root=PARTUUID={p.uuid}" for p in partitions if p.type.startswith("root")), None) - if not root: - root = next((f"mount.usr=PARTUUID={p.uuid}" for p in partitions if p.type.startswith("usr")), None) - + root = finalize_root(partitions) assert root initrd = build_initrd(state) @@ -981,23 +956,27 @@ def build_kernel_modules_initrd(state: MkosiState, kver: str) -> Path: return kmods -def finalize_roothash(partitions: Sequence[Partition]) -> Optional[str]: - roothash = usrhash = None - - for p in partitions: - if (h := p.roothash) is None: - continue - - if not (p.type.startswith("usr") or p.type.startswith("root")): - die(f"Found roothash property on unexpected partition type {p.type}") - - # When there's multiple verity enabled root or usr partitions, the first one wins. - if p.type.startswith("usr"): - usrhash = usrhash or h - else: - roothash = roothash or h +def extract_pe_section(state: MkosiState, binary: Path, section: str, output: Path) -> None: + # If there's no tools tree, prefer the interpreter from MKOSI_INTERPRETER. If there is a tools + # tree, just use the default python3 interpreter. + python = "python3" if state.config.tools_tree else os.getenv("MKOSI_INTERPRETER", "python3") + + # When using a tools tree, we want to use the pefile module from the tools tree instead of requiring that + # python-pefile is installed on the host. So we execute python as a subprocess to make sure we load + # pefile from the tools tree if one is used. + + # TODO: Use ignore_padding=True instead of length once we can depend on a newer pefile. + pefile = textwrap.dedent( + f"""\ + import pefile + from pathlib import Path + pe = pefile.PE("{binary}", fast_load=True) + section = {{s.Name.decode().strip("\\0"): s for s in pe.sections}}["{section}"] + Path("{output}").write_bytes(section.get_data(length=section.Misc_VirtualSize)) + """ + ) - return f"roothash={roothash}" if roothash else f"usrhash={usrhash}" if usrhash else None + run([python], input=pefile) def install_unified_kernel(state: MkosiState, partitions: Sequence[Partition]) -> None: @@ -1128,32 +1107,17 @@ def install_unified_kernel(state: MkosiState, partitions: Sequence[Partition]) - run(cmd) + if not (state.staging / state.config.output_split_initrd).exists(): + # Extract the combined initrds from the UKI so we can use it to direct kernel boot with qemu + # if needed. + extract_pe_section(state, boot_binary, ".initrd", state.staging / state.config.output_split_initrd) + if not (state.staging / state.config.output_split_uki).exists(): shutil.copy(boot_binary, state.staging / state.config.output_split_uki) # ukify will have signed the kernel image as well. Let's make sure we put the signed kernel # image in the output directory instead of the unsigned one by reading it from the UKI. - - # When using a tools tree, we want to use the pefile module from the tools tree instead of - # requiring that python-pefile is installed on the host. So we execute python as a subprocess - # to make sure we load pefile from the tools tree if one is used. - - # TODO: Use ignore_padding=True instead of length once we can depend on a newer pefile. - pefile = textwrap.dedent( - f"""\ - import pefile - from pathlib import Path - pe = pefile.PE("{boot_binary}", fast_load=True) - linux = {{s.Name.decode().strip("\\0"): s for s in pe.sections}}[".linux"] - (Path("{state.staging}") / "{state.config.output_split_kernel}").write_bytes(linux.get_data(length=linux.Misc_VirtualSize)) - """ - ) - - # If there's no tools tree, prefer the interpreter from MKOSI_INTERPRETER. If there is a - # tools tree, just use the default python3 interpreter. - python = "python3" if state.config.tools_tree else os.getenv("MKOSI_INTERPRETER", "python3") - - run([python], input=pefile) + extract_pe_section(state, boot_binary, ".linux", state.staging / state.config.output_split_kernel) print_output_size(boot_binary) diff --git a/mkosi/config.py b/mkosi/config.py index 231886cb3..399063dd4 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -773,6 +773,10 @@ class MkosiConfig: def output_split_kernel(self) -> str: return f"{self.output_with_version}.vmlinuz" + @property + def output_split_initrd(self) -> str: + return f"{self.output_with_version}.initrd" + @property def output_nspawn_settings(self) -> str: return f"{self.output_with_version}.nspawn" diff --git a/mkosi/partition.py b/mkosi/partition.py new file mode 100644 index 000000000..13c51f6a8 --- /dev/null +++ b/mkosi/partition.py @@ -0,0 +1,65 @@ +import dataclasses +import json +import subprocess +from collections.abc import Sequence +from pathlib import Path +from typing import Any, Mapping, Optional + +from mkosi.log import die +from mkosi.run import run + + +@dataclasses.dataclass(frozen=True) +class Partition: + type: str + uuid: str + partno: Optional[int] + split_path: Optional[Path] + roothash: Optional[str] + + @classmethod + def from_dict(cls, dict: Mapping[str, Any]) -> "Partition": + return cls( + type=dict["type"], + uuid=dict["uuid"], + partno=int(partno) if (partno := dict.get("partno")) else None, + split_path=Path(p) if ((p := dict.get("split_path")) and p != "-") else None, + roothash=dict.get("roothash"), + ) + + GRUB_BOOT_PARTITION_UUID = "21686148-6449-6e6f-744e-656564454649" + + +def find_partitions(image: Path) -> list[Partition]: + output = json.loads(run(["systemd-repart", "--json=short", image], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout) + return [Partition.from_dict(d) for d in output] + + +def finalize_roothash(partitions: Sequence[Partition]) -> Optional[str]: + roothash = usrhash = None + + for p in partitions: + if (h := p.roothash) is None: + continue + + if not (p.type.startswith("usr") or p.type.startswith("root")): + die(f"Found roothash property on unexpected partition type {p.type}") + + # When there's multiple verity enabled root or usr partitions, the first one wins. + if p.type.startswith("usr"): + usrhash = usrhash or h + else: + roothash = roothash or h + + return f"roothash={roothash}" if roothash else f"usrhash={usrhash}" if usrhash else None + + +def finalize_root(partitions: Sequence[Partition]) -> Optional[str]: + root = finalize_roothash(partitions) + if not root: + root = next((f"root=PARTUUID={p.uuid}" for p in partitions if p.type.startswith("root")), None) + if not root: + root = next((f"mount.usr=PARTUUID={p.uuid}" for p in partitions if p.type.startswith("usr")), None) + + return root diff --git a/mkosi/qemu.py b/mkosi/qemu.py index 6cf5d8210..0557dab75 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -24,6 +24,7 @@ from mkosi.config import ( QemuFirmware, ) from mkosi.log import die +from mkosi.partition import finalize_root, find_partitions from mkosi.run import MkosiAsyncioThread, run, spawn from mkosi.tree import copy_tree, rmtree from mkosi.types import PathString @@ -324,11 +325,23 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig) -> None: if kernel: cmdline += ["-kernel", kernel] - cmdline += ["-append", " ".join(config.kernel_command_line + config.kernel_command_line_extra)] + if config.output_format == OutputFormat.disk: + # We can't rely on gpt-auto-generator when direct kernel booting so synthesize a root= + # kernel argument instead. + root = finalize_root(find_partitions(fname)) + if not root: + die("Cannot direct kernel boot image without root or usr partition") + else: + root = "" + + cmdline += ["-append", " ".join([root] + config.kernel_command_line + config.kernel_command_line_extra)] if config.output_format == OutputFormat.cpio: cmdline += ["-initrd", fname] else: + if firmware == QemuFirmware.direct: + cmdline += ["-initrd", config.output_dir / config.output_split_initrd] + cmdline += ["-drive", f"if=none,id=mkosi,file={fname},format=raw", "-device", "virtio-scsi-pci,id=scsi", "-device", f"scsi-{'cd' if config.qemu_cdrom else 'hd'},drive=mkosi,bootindex=1"] -- 2.47.2