From: Paymon MARANDI Date: Sat, 20 May 2023 12:38:21 +0000 (-0400) Subject: gentoo: support initrd X-Git-Tag: v15~103^2~30 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=49d72ce5c1d0e2764d9baef1ed4bcc3679a1750b;p=thirdparty%2Fmkosi.git gentoo: support initrd Signed-off-by: Paymon MARANDI --- diff --git a/mkosi/distributions/gentoo.py b/mkosi/distributions/gentoo.py index 9a4ffaec0..ae0dc49f8 100644 --- a/mkosi/distributions/gentoo.py +++ b/mkosi/distributions/gentoo.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: LGPL-2.1+ +from dataclasses import dataclass import logging import os import re @@ -8,6 +9,7 @@ import urllib.request from collections.abc import Sequence from pathlib import Path from textwrap import dedent +from typing import List, Optional from mkosi.architecture import Architecture from mkosi.distributions import DistributionInstaller @@ -20,11 +22,23 @@ from mkosi.state import MkosiState def invoke_emerge( state: MkosiState, + sysroot: Optional[Path] = None, + root: Optional[Path] = None, + bwrap_params: list[str] = [], pkgs: Sequence[str] = (), actions: Sequence[str] = (), opts: Sequence[str] = (), + env: dict[str, str] = {}, ) -> None: jobs = os.cpu_count() or 1 + if sysroot is not None: + target_root_mntp = Path("/tmp/mkosi-root") + bwrap_params += ["--bind", state.root, target_root_mntp] + root = target_root_mntp + else: + sysroot = state.root + root = None + emerge_default_opts = [ "--buildpkg=y", "--usepkg=y", @@ -32,36 +46,62 @@ def invoke_emerge( f"--jobs={jobs}", f"--load-average={jobs+1}", "--nospinner", + *([f"--root={root}"] if root else []), ] if ARG_DEBUG.get(): emerge_default_opts += ["--verbose", "--quiet=n", "--quiet-fail=n"] else: emerge_default_opts += ["--quiet-build", "--quiet"] cmd = ["emerge", *pkgs, *emerge_default_opts, *opts, *actions] - bwrap_params = [ + bwrap_params += [ "--bind", state.cache / "binpkgs", "/var/cache/binpkgs", "--bind", state.cache / "distfiles", "/var/cache/distfiles", "--bind", state.cache / "repos", "/var/db/repos", ] - run_workspace_command(state.root, cmd, bwrap_params=bwrap_params, - network=True, env=state.environment) + run_workspace_command(sysroot, cmd, bwrap_params=bwrap_params, + network=True, env=env) + + +@dataclass +class GentooAtoms: + stage2: List[str] + bare_minimal: List[str] + boot: List[str] + system: List[str] class Gentoo: arch_profile: Path arch: str - custom_profile_path: Path - ebuild_sh_env_dir: Path emerge_vars: dict[str, str] portage_cfg_dir: Path - binrepo_cfg_file: Path - profile_path: Path root: Path - pkgs: dict[str, list[str]] = {} + stage3_cache: Path + stage3_tar_path: Path + pkgs: GentooAtoms = GentooAtoms( + stage2=[ + "sys-apps/baselayout", + "sys-apps/util-linux", + "app-crypt/libb2", + ], + bare_minimal=[ + "app-alternatives/sh", + "sys-apps/shadow", + "app-admin/eselect", + "net-misc/iputils", + "sys-apps/coreutils", + "sys-apps/portage", + ], + boot=[ + "sys-apps/systemd", + "sys-kernel/dracut", + "sys-kernel/gentoo-kernel-bin", + ], + system=["@system"] + ) EMERGE_UPDATE_OPTS = [ "--update", - "--tree", "--changed-use", "--newuse", "--deep", @@ -72,7 +112,6 @@ class Gentoo: portage_use_flags = [ "initramfs", - "git", # for sync-type=git "symlink", # for kernel ] @@ -89,6 +128,9 @@ class Gentoo: "parallel-install", "getbinpkg", "-candy", + "noman", + "nodoc", + "noinfo", ] @staticmethod @@ -131,9 +173,7 @@ class Gentoo: """ try: from portage.const import ( # type: ignore - CUSTOM_PROFILE_PATH, EBUILD_SH_ENV_DIR, - PROFILE_PATH, USER_CONFIG_PATH, BINREPOS_CONF_FILE, ) @@ -142,11 +182,9 @@ class Gentoo: logging.info(PORTAGE_INSTALL_INSTRUCTIONS) raise e - return dict(profile_path=PROFILE_PATH, - custom_profile_path=CUSTOM_PROFILE_PATH, - ebuild_sh_env_dir=EBUILD_SH_ENV_DIR, + return dict(ebuild_sh_env_dir=EBUILD_SH_ENV_DIR, portage_cfg_dir=USER_CONFIG_PATH, - binrepo_cfg_file=BINREPOS_CONF_FILE) + binrepos_conf_file=BINREPOS_CONF_FILE) @complete_step("Installing Gentoo…") def __init__(self, state: MkosiState) -> None: @@ -154,35 +192,22 @@ class Gentoo: self.state = state self.config = self.state.config self.root = self.state.root - ret = self.try_import_portage() + self.portage_consts = self.try_import_portage() from portage.package.ebuild.config import config as portage_cfg # type: ignore - self.portage_cfg = portage_cfg(config_root=str(state.root), - target_root=str(state.root), - sysroot=str(state.root), eprefix=None) + self.portage_cfg = portage_cfg() PORTAGE_MISCONFIGURED_MSG = "Missing defaults for portage, bailing out" # we check for PORTDIR, but we could check for any other one if self.portage_cfg['PORTDIR'] is None: die(PORTAGE_MISCONFIGURED_MSG) - self.profile_path = state.root / ret["profile_path"] - self.custom_profile_path = state.root / ret["custom_profile_path"] - self.ebuild_sh_env_dir = state.root / ret["ebuild_sh_env_dir"] - self.portage_cfg_dir = state.root / ret["portage_cfg_dir"] - self.binrepo_cfg_file = state.root / ret["binrepo_cfg_file"] - - self.portage_cfg_dir.mkdir(parents=True, exist_ok=True) - self.arch, _ = state.installer.architecture(state.config.architecture) self.arch_profile = Path(f"default/linux/{self.arch}/{state.config.release}/no-multilib/systemd/merged-usr") - self.pkgs['sys'] = ["@world"] + self.get_current_stage3() - self.pkgs['boot'] = [ - "sys-kernel/installkernel-systemd-boot", - "sys-kernel/gentoo-kernel-bin", - ] + self.portage_cfg_dir = self.stage3_cache / self.portage_consts["portage_cfg_dir"] self.emerge_vars = { "FEATURES": " ".join(self.portage_features), @@ -193,13 +218,12 @@ class Gentoo: self.state.cache.joinpath(d).mkdir(exist_ok=True) self.fetch_fix_stage3() - self.set_useflags() - self.mkosi_conf() + self.sync_profiles() self.get_snapshot_of_portage_tree() - self.update_stage3() - self.depclean() - def fetch_fix_stage3(self) -> None: + self.merge_system() + + def get_current_stage3(self) -> None: """usrmerge tracker bug: https://bugs.gentoo.org/690294""" # http://distfiles.gentoo.org/releases/amd64/autobuilds/latest-stage3.txt @@ -219,45 +243,47 @@ class Gentoo: all_lines = r.readlines() for line in all_lines: if (m := re.match(regexp, line.decode("utf-8"))): - stage3_tar = Path(m.group(0)) + self.stage3_tar = Path(m.group(0)) break else: die("profile names changed upstream?") + self.stage3_tar_path = self.state.cache_dir / self.stage3_tar + self.stage3_cache = self.stage3_tar_path.with_name(self.stage3_tar.name).with_suffix(".tmp") + + def fetch_fix_stage3(self) -> None: stage3_url_path = urllib.parse.urljoin( self.portage_cfg["GENTOO_MIRRORS"], - f"releases/{self.arch}/autobuilds/{stage3_tar}", + f"releases/{self.arch}/autobuilds/{self.stage3_tar}", ) - stage3_tar_path = self.state.cache_dir / stage3_tar - stage3_tmp_extract = stage3_tar_path.with_name(stage3_tar.name + ".tmp") - if not stage3_tar_path.is_file(): + if not self.stage3_tar_path.is_file(): log_step(f"Fetching {stage3_url_path}") - stage3_tar_path.parent.mkdir(parents=True, exist_ok=True) - urllib.request.urlretrieve(stage3_url_path, stage3_tar_path) + self.stage3_tar_path.parent.mkdir(parents=True, exist_ok=True) + urllib.request.urlretrieve(stage3_url_path, self.stage3_tar_path) - stage3_tmp_extract.mkdir(parents=True, exist_ok=True) + self.stage3_cache.mkdir(parents=True, exist_ok=True) - with flock(stage3_tmp_extract): - if not stage3_tmp_extract.joinpath(".cache_isclean").exists(): - log_step(f"Extracting {stage3_tar.name} to {stage3_tmp_extract}") + with flock(self.stage3_cache): + if not self.stage3_cache.joinpath(".cache_isclean").exists(): + log_step(f"Extracting {self.stage3_tar.name} to {self.stage3_cache}") run([ "tar", "--numeric-owner", - "-C", stage3_tmp_extract, + "-C", self.stage3_cache, "--extract", - "--file", stage3_tar_path, + "--file", self.stage3_tar_path, "--exclude", "./dev", ]) - unlink_try_hard(stage3_tmp_extract.joinpath("dev")) - unlink_try_hard(stage3_tmp_extract.joinpath("proc")) - unlink_try_hard(stage3_tmp_extract.joinpath("sys")) + unlink_try_hard(self.stage3_cache.joinpath("dev")) + unlink_try_hard(self.stage3_cache.joinpath("proc")) + unlink_try_hard(self.stage3_cache.joinpath("sys")) - stage3_tmp_extract.joinpath(".cache_isclean").touch() + self.stage3_cache.joinpath(".cache_isclean").touch() - log_step(f"Copying {stage3_tmp_extract} to {self.root}") - copy_path(stage3_tmp_extract, self.root) + self.set_useflags() + self.mkosi_conf() def set_useflags(self) -> None: package_use = self.portage_cfg_dir / "package.use" @@ -284,11 +310,21 @@ class Gentoo: """ ) ) + if self.state.config.make_initrd: + package_use.joinpath("minimal").write_text( + dedent( + """\ + # MKOSI + */* minimal + """ + ) + ) def mkosi_conf(self) -> None: package_env = self.portage_cfg_dir / "package.env" package_env.mkdir(exist_ok=True) - self.ebuild_sh_env_dir.mkdir(exist_ok=True) + ebuild_sh_env_dir = self.stage3_cache / self.portage_consts["ebuild_sh_env_dir"] + ebuild_sh_env_dir.mkdir(exist_ok=True) # apply whatever we put in mkosi_conf to runs invocation of emerge package_env.joinpath("mkosi.conf").write_text("*/* mkosi.conf\n") @@ -298,7 +334,7 @@ class Gentoo: emerge_vars_str = "" emerge_vars_str += "\n".join(f'{k}="${{{k}}} {v}"' for k, v in self.emerge_vars.items()) - self.ebuild_sh_env_dir.joinpath("mkosi.conf").write_text( + ebuild_sh_env_dir.joinpath("mkosi.conf").write_text( dedent( f"""\ # MKOSI: these were used during image creation... @@ -308,7 +344,8 @@ class Gentoo: ) ) - with self.binrepo_cfg_file.open(mode='a') as f: + repos_cfg = self.stage3_cache / self.portage_consts["binrepos_conf_file"] + with repos_cfg.open(mode='a') as f: for repo in self.config.repositories: f.write( dedent( @@ -320,19 +357,49 @@ class Gentoo: ) ) + def sync_profiles(self) -> None: + root_portage_cfg = self.root + root_portage_cfg /= self.portage_cfg_dir.relative_to(self.stage3_cache) + root_portage_cfg.mkdir(parents=True, exist_ok=True) + copy_path(self.portage_cfg_dir, root_portage_cfg) + def get_snapshot_of_portage_tree(self) -> None: bwrap_params = ["--bind", self.state.cache / "repos", "/var/db/repos"] - run_workspace_command(self.state.root, ["/usr/bin/emerge-webrsync"], - bwrap_params=bwrap_params, network=True, - env=self.state.environment) - - def update_stage3(self) -> None: - invoke_emerge(self.state, opts=self.EMERGE_UPDATE_OPTS, pkgs=self.pkgs['boot']) - invoke_emerge(self.state, opts=["--config"], pkgs=["sys-kernel/gentoo-kernel-bin"]) - invoke_emerge(self.state, opts=self.EMERGE_UPDATE_OPTS, pkgs=self.pkgs['sys']) - - def depclean(self) -> None: - invoke_emerge(self.state, actions=["--depclean"]) + run_workspace_command(self.stage3_cache, ["/usr/bin/emerge-webrsync"], + bwrap_params=bwrap_params, network=True) + + def merge_system(self) -> None: + opts = [ + "--with-bdeps=n", + "--complete-graph-if-new-use=y", + "--verbose-conflicts", + "--changed-use", + "--newuse", + ] + env = {} + env.update(self.emerge_vars) + env.update({"USE": f"{self.emerge_vars['USE']} build"}) + with complete_step("merging stage2"): + invoke_emerge(self.state, sysroot=self.stage3_cache, + opts=opts+["--emptytree", "--nodeps"], + pkgs=self.pkgs.stage2, env=env) + opts += ["--noreplace", "--root-deps=rdeps"] + with complete_step("Merging bare minimal atoms"): + invoke_emerge(self.state, sysroot=self.stage3_cache, + opts=opts+["--exclude", "sys-devel/*"], + pkgs=self.pkgs.bare_minimal, env=self.emerge_vars) + with complete_step("Merging atoms required for boot"): + invoke_emerge(self.state, sysroot=self.stage3_cache, + opts=opts+["--exclude", "sys-devel/*"], + pkgs=self.pkgs.boot) + invoke_emerge(self.state, actions=["--config"], + pkgs=["sys-kernel/gentoo-kernel-bin"]) + if self.state.config.make_initrd: + return + + invoke_emerge(self.state, sysroot=self.stage3_cache, + opts=opts+["--exclude", "sys-devel/*"], + pkgs=self.pkgs.system) class GentooInstaller(DistributionInstaller): @@ -356,7 +423,7 @@ class GentooInstaller(DistributionInstaller): @classmethod def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: - invoke_emerge(state, packages) + invoke_emerge(state, opts=["--noreplace"], pkgs=packages) @staticmethod def architecture(arch: Architecture) -> str: