From: Daan De Meyer Date: Thu, 9 Nov 2023 11:05:36 +0000 (+0100) Subject: Add esp output format X-Git-Tag: v19~8^2 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F2057%2Fhead;p=thirdparty%2Fmkosi.git Add esp output format This is largely the same as the uki output format, but additionally we wrap the uki in a disk image with just an ESP. --- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dedc7a71..6edc3e845 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,9 +134,12 @@ jobs: - cpio - disk - uki + - esp exclude: - distro: rhel-ubi format: uki + - distro: rhel-ubi + format: esp steps: - uses: actions/checkout@v3 @@ -182,7 +185,7 @@ jobs: run: sudo mkosi --debug boot - name: Boot ${{ matrix.distro }}/${{ matrix.format }} QEMU - if: matrix.distro != 'rhel-ubi' && (matrix.format == 'disk' || matrix.format == 'uki' || matrix.format == 'cpio') + if: matrix.distro != 'rhel-ubi' && matrix.format != 'directory' && matrix.format != 'tar' run: timeout -k 30 10m mkosi --debug qemu - name: Boot ${{ matrix.distro }}/${{ matrix.format }} BIOS diff --git a/mkosi/__init__.py b/mkosi/__init__.py index c8833aa4b..5f3b17b05 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -1319,7 +1319,7 @@ def want_uki(config: MkosiConfig) -> bool: # Note that this returns True also in the case where autodetection might later # cause the UKI not to be installed after the file system has been populated. - if config.output_format == OutputFormat.uki: + if config.output_format in (OutputFormat.uki, OutputFormat.esp): return True if config.bootable == ConfigFeature.disabled: @@ -1344,7 +1344,7 @@ def install_uki(state: MkosiState, partitions: Sequence[Partition]) -> None: # benefit that they can be signed like normal EFI binaries, and can encode everything necessary to boot a # specific root device, including the root hash. - if not want_uki(state.config) or state.config.output_format == OutputFormat.uki: + if not want_uki(state.config) or state.config.output_format in (OutputFormat.uki, OutputFormat.esp): return arch = state.config.architecture.to_efi() @@ -1682,7 +1682,7 @@ def check_tools(args: MkosiArgs, config: MkosiConfig) -> None: hint="Bootable=no can be used to create a non-bootable image", ) - if config.output_format == OutputFormat.disk: + if config.output_format in (OutputFormat.disk, OutputFormat.esp): check_systemd_tool("systemd-repart", reason="build disk images") if args.verb == Verb.boot: @@ -1956,10 +1956,14 @@ def reuse_cache(state: MkosiState) -> bool: return True -def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: bool = False) -> list[Partition]: - if not state.config.output_format == OutputFormat.disk: - return [] - +def make_image( + state: MkosiState, + msg: str, + skip: Sequence[str] = [], + split: bool = False, + root: Optional[Path] = None, + definitions: Sequence[Path] = [], +) -> list[Partition]: cmdline: list[PathString] = [ "systemd-repart", "--empty=allow", @@ -1968,11 +1972,12 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo "--json=pretty", "--no-pager", "--offline=yes", - "--root", state.root, "--seed", str(state.config.seed) if state.config.seed else "random", state.staging / state.config.output_with_format, ] + if root: + cmdline += ["--root", root] if not state.config.architecture.is_native(): cmdline += ["--architecture", str(state.config.architecture)] if not (state.staging / state.config.output_with_format).exists(): @@ -1990,19 +1995,52 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo if state.config.sector_size: cmdline += ["--sector-size", state.config.sector_size] - if state.config.repart_dirs: - for d in state.config.repart_dirs: + if definitions: + for d in definitions: cmdline += ["--definitions", d] # Subvolumes= only works with --offline=no. - grep = run(["grep", "--recursive", "--include=*.conf", "Subvolumes=", *state.config.repart_dirs], + grep = run(["grep", "--recursive", "--include=*.conf", "Subvolumes=", *definitions], stdout=subprocess.DEVNULL, check=False) if grep.returncode == 0: cmdline += ["--offline=no"] + + env = { + option: value + for option, value in state.config.environment.items() + if option.startswith("SYSTEMD_REPART_MKFS_OPTIONS_") or option == "SOURCE_DATE_EPOCH" + } + + with complete_step(msg): + output = json.loads(run(cmdline, stdout=subprocess.PIPE, env=env).stdout) + + logging.debug(json.dumps(output, indent=4)) + + partitions = [Partition.from_dict(d) for d in output] + + if split: + for p in partitions: + if p.split_path: + maybe_compress(state.config, state.config.compress_output, p.split_path) + + return partitions + + +def make_disk( + state: MkosiState, + msg: str, + skip: Sequence[str] = [], + split: bool = False, +) -> list[Partition]: + if state.config.output_format != OutputFormat.disk: + return [] + + if state.config.repart_dirs: + definitions = state.config.repart_dirs else: - definitions = state.workspace / "repart-definitions" - if not definitions.exists(): - definitions.mkdir() + defaults = state.workspace / "repart-definitions" + if not defaults.exists(): + defaults.mkdir() if (arch := state.config.architecture.to_efi()): bootloader = state.root / f"efi/EFI/BOOT/BOOT{arch.upper()}.EFI" else: @@ -2012,7 +2050,7 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo bios = (state.config.bootable != ConfigFeature.disabled and want_grub_bios(state)) if bios: - (definitions / "05-bios.conf").write_text( + (defaults / "05-bios.conf").write_text( textwrap.dedent( f"""\ [Partition] @@ -2032,7 +2070,7 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo # Even if we're doing BIOS, let's still use the ESP to store the kernels, initrds and grub # modules. We cant use UKIs so we have to put each kernel and initrd on the ESP twice, so # let's make the ESP twice as big in that case. - (definitions / "00-esp.conf").write_text( + (defaults / "00-esp.conf").write_text( textwrap.dedent( f"""\ [Partition] @@ -2045,7 +2083,7 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo ) ) - (definitions / "10-root.conf").write_text( + (defaults / "10-root.conf").write_text( textwrap.dedent( f"""\ [Partition] @@ -2057,27 +2095,38 @@ def make_image(state: MkosiState, msg: str, skip: Sequence[str] = [], split: boo ) ) - cmdline += ["--definitions", definitions] + definitions = [defaults] - env = { - option: value - for option, value in state.config.environment.items() - if option.startswith("SYSTEMD_REPART_MKFS_OPTIONS_") or option == "SOURCE_DATE_EPOCH" - } + return make_image(state, msg=msg, skip=skip, split=split, root=state.root, definitions=definitions) - with complete_step(msg): - output = json.loads(run(cmdline, stdout=subprocess.PIPE, env=env).stdout) - logging.debug(json.dumps(output, indent=4)) +def make_esp(state: MkosiState, uki: Path) -> list[Partition]: + if not (arch := state.config.architecture.to_efi()): + die(f"Architecture {state.config.architecture} does not support UEFI") - partitions = [Partition.from_dict(d) for d in output] + definitions = state.workspace / "esp-definitions" + definitions.mkdir(exist_ok=True) - if split: - for p in partitions: - if p.split_path: - maybe_compress(state.config, state.config.compress_output, p.split_path) + # Use a minimum of 512MB because otherwise the generated FAT filesystem will have too few clusters to be considered + # a FAT32 filesystem by OVMF which will refuse to boot from it. Always reserve 10MB for filesystem metadata. + size = max(uki.stat().st_size, 502 * 1024**2) + 10 * 1024**2 - return partitions + # TODO: Remove the extra 4096 for the max size once https://github.com/systemd/systemd/pull/29954 is in a stable + # release. + (definitions / "00-esp.conf").write_text( + textwrap.dedent( + f"""\ + [Partition] + Type=esp + Format=vfat + CopyFiles={uki}:/EFI/BOOT/BOOT{arch.upper()}.EFI + SizeMinBytes={size} + SizeMaxBytes={size + 4096} + """ + ) + ) + + return make_image(state, msg="Generating ESP image", definitions=[definitions]) def finalize_staging(state: MkosiState) -> None: @@ -2192,17 +2241,17 @@ def build_image(args: MkosiArgs, config: MkosiConfig) -> None: run_finalize_scripts(state) normalize_mtime(state.root, state.config.source_date_epoch) - partitions = make_image(state, skip=("esp", "xbootldr"), msg="Generating disk image") + partitions = make_disk(state, skip=("esp", "xbootldr"), msg="Generating disk image") install_uki(state, partitions) prepare_grub_efi(state) prepare_grub_bios(state, partitions) normalize_mtime(state.root, state.config.source_date_epoch, directory=Path("boot")) normalize_mtime(state.root, state.config.source_date_epoch, directory=Path("efi")) - partitions = make_image(state, msg="Formatting ESP/XBOOTLDR partitions") + partitions = make_disk(state, msg="Formatting ESP/XBOOTLDR partitions") install_grub_bios(state, partitions) if state.config.split_artifacts: - make_image(state, split=True, msg="Extracting partitions") + make_disk(state, split=True, msg="Extracting partitions") copy_vmlinuz(state) @@ -2212,10 +2261,13 @@ def build_image(args: MkosiArgs, config: MkosiConfig) -> None: make_cpio(state.root, state.staging / state.config.output_with_format) elif state.config.output_format == OutputFormat.uki: make_uki(state, state.staging / state.config.output_with_format) + elif state.config.output_format == OutputFormat.esp: + make_uki(state, state.staging / state.config.output_split_uki) + make_esp(state, state.staging / state.config.output_split_uki) elif state.config.output_format == OutputFormat.directory: state.root.rename(state.staging / state.config.output_with_format) - if config.output_format != OutputFormat.uki: + if config.output_format not in (OutputFormat.uki, OutputFormat.esp): maybe_compress(state.config, state.config.compress_output, state.staging / state.config.output_with_format, state.staging / state.config.output_with_compression) diff --git a/mkosi/config.py b/mkosi/config.py index 93be0eb0c..ae78396fb 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -104,11 +104,13 @@ class OutputFormat(StrEnum): cpio = enum.auto() disk = enum.auto() uki = enum.auto() + esp = enum.auto() none = enum.auto() def extension(self) -> str: return { OutputFormat.disk: ".raw", + OutputFormat.esp: ".raw", OutputFormat.cpio: ".cpio", OutputFormat.tar: ".tar", OutputFormat.uki: ".efi", @@ -302,7 +304,7 @@ def config_parse_source_date_epoch(value: Optional[str], old: Optional[int]) -> def config_default_compression(namespace: argparse.Namespace) -> Compression: - if namespace.output_format in (OutputFormat.cpio, OutputFormat.uki): + if namespace.output_format in (OutputFormat.cpio, OutputFormat.uki, OutputFormat.esp): if namespace.distribution.is_centos_variant() and int(namespace.release) <= 8: return Compression.xz else: @@ -952,7 +954,7 @@ class MkosiConfig: def output_with_compression(self) -> str: output = self.output_with_format - if self.compress_output and self.output_format != OutputFormat.uki: + if self.compress_output and self.output_format not in (OutputFormat.uki, OutputFormat.esp): output += f".{self.compress_output}" return output @@ -2894,7 +2896,7 @@ Clean Package Manager Metadata: {yes_no_auto(config.clean_package_metadata)} SSH: {yes_no(config.ssh)} """ - if config.output_format == OutputFormat.disk: + if config.output_format in (OutputFormat.disk, OutputFormat.uki, OutputFormat.esp): summary += f"""\ {bold("VALIDATION")}: diff --git a/mkosi/qemu.py b/mkosi/qemu.py index 12d49a57a..69c645379 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -366,11 +366,17 @@ def qemu_version(config: MkosiConfig) -> GenericVersion: def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[QemuDeviceNode, int]) -> None: - if config.output_format not in (OutputFormat.disk, OutputFormat.cpio, OutputFormat.uki, OutputFormat.directory): + if config.output_format not in ( + OutputFormat.disk, + OutputFormat.cpio, + OutputFormat.uki, + OutputFormat.esp, + OutputFormat.directory, + ): die(f"{config.output_format} images cannot be booted in qemu") if ( - config.output_format in (OutputFormat.cpio, OutputFormat.uki) and + config.output_format in (OutputFormat.cpio, OutputFormat.uki, OutputFormat.esp) and config.qemu_firmware not in (QemuFirmware.auto, QemuFirmware.linux, QemuFirmware.uefi) ): die(f"{config.output_format} images cannot be booted with the '{config.qemu_firmware}' firmware") @@ -396,7 +402,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[Qemu else: kernel = None - if config.output_format == OutputFormat.uki and kernel: + if config.output_format in (OutputFormat.uki, OutputFormat.esp) and kernel: logging.warning( f"Booting UKI output, kernel {kernel} configured with QemuKernel= or passed with -kernel will not be used" ) @@ -514,8 +520,8 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[Qemu "-drive", f"file={ovmf_vars.name},if=pflash,format=raw", ] - if config.qemu_cdrom and config.output_format == OutputFormat.disk: - # CD-ROM devices have sector size 2048 so we transform the disk image into one with sector size 2048. + if config.qemu_cdrom and config.output_format in (OutputFormat.disk, OutputFormat.esp): + # CD-ROM devices have sector size 2048 so we transform disk images into ones with sector size 2048. src = (config.output_dir_or_cwd() / config.output).resolve() fname = src.parent / f"{src.name}-{uuid.uuid4().hex}" run(["systemd-repart", @@ -600,7 +606,7 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[Qemu ): cmdline += ["-initrd", config.output_dir_or_cwd() / config.output_split_initrd] - if config.output_format == OutputFormat.disk: + if config.output_format in (OutputFormat.disk, OutputFormat.esp): cmdline += ["-drive", f"if=none,id=mkosi,file={fname},format=raw", "-device", "virtio-scsi-pci,id=scsi", "-device", f"scsi-{'cd' if config.qemu_cdrom else 'hd'},drive=mkosi,bootindex=1"] diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 94718651d..6bfeb37a7 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -557,12 +557,14 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `Format=`, `--format=`, `-t` -: The image format type to generate. One of `directory` (for generating an OS - image directly in a local directory), `tar` (similar, but a tarball of the OS - image is generated), `cpio` (similar, but a cpio archive is generated), - `disk` (a block device OS image with a GPT partition table), `uki` (a unified - kernel image with the OS image in the `.initrd` PE section) or `none` (the OS - image is solely intended as a build image to produce another artifact). +: The image format type to generate. One of `directory` (for generating + an OS image directly in a local directory), `tar` (similar, but a + tarball of the OS image is generated), `cpio` (similar, but a cpio + archive is generated), `disk` (a block device OS image with a GPT + partition table), `uki` (a unified kernel image with the OS image in + the `.initrd` PE section), `esp` (`uki` but wrapped in a disk image + with only an ESP partition) or `none` (the OS image is solely intended + as a build image to produce another artifact). : If the `disk` output format is used, the disk image is generated using `systemd-repart`. The repart partition definition files to use can be