From: Daan De Meyer Date: Thu, 13 Apr 2023 12:19:12 +0000 (+0200) Subject: Streamline package installation X-Git-Tag: v15~251^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7efe16a7c00142f0964233e7401d0daada8d8fbe;p=thirdparty%2Fmkosi.git Streamline package installation Let's move most of the logic related to installing packages into install_packages() for each distribution, and only keep the first time installation stuff in install(). This allows us to move most of the base_image checks out of the distribution specific files into a single check in install_distribution() where if a base image is provided, we call install_packages(). Otherwise, we call install(). --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 036c21088..90c249301 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -274,7 +274,15 @@ def install_distribution(state: MkosiState, cached: bool) -> None: if cached: return - state.installer.install(state) + if state.config.base_image: + if not state.config.packages: + return + + with complete_step(f"Installing extra packages for {str(state.config.distribution).capitalize()}"): + state.installer.install_packages(state, state.config.packages) + else: + with complete_step(f"Installing {str(state.config.distribution).capitalize()}"): + state.installer.install(state) def install_build_packages(state: MkosiState, cached: bool) -> None: diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index 6a309ed24..865e2651c 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -5,7 +5,6 @@ from textwrap import dedent from mkosi.backend import MkosiState, sort_packages from mkosi.distributions import DistributionInstaller -from mkosi.log import complete_step from mkosi.run import run_with_apivfs from mkosi.types import PathString @@ -17,76 +16,72 @@ class ArchInstaller(DistributionInstaller): @classmethod def install(cls, state: MkosiState) -> None: - return install_arch(state) + cls.install_packages(state, ["filesystem", *state.config.packages]) @classmethod def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: - return invoke_pacman(state, packages) + assert state.config.mirror + if state.config.local_mirror: + server = f"Server = {state.config.local_mirror}" + else: + if state.config.architecture == "aarch64": + server = f"Server = {state.config.mirror}/$arch/$repo" + else: + server = f"Server = {state.config.mirror}/$repo/os/$arch" -@complete_step("Installing Arch Linux…") -def install_arch(state: MkosiState) -> None: - assert state.config.mirror + # Create base layout for pacman and pacman-key + state.root.joinpath("var/lib/pacman").mkdir(mode=0o755, exist_ok=True, parents=True) - if state.config.local_mirror: - server = f"Server = {state.config.local_mirror}" - else: - if state.config.architecture == "aarch64": - server = f"Server = {state.config.mirror}/$arch/$repo" + pacman_conf = state.workspace / "pacman.conf" + if state.config.repository_key_check: + sig_level = "Required DatabaseOptional" else: - server = f"Server = {state.config.mirror}/$repo/os/$arch" - - # Create base layout for pacman and pacman-key - state.root.joinpath("var/lib/pacman").mkdir(mode=0o755, exist_ok=True, parents=True) - - pacman_conf = state.workspace / "pacman.conf" - if state.config.repository_key_check: - sig_level = "Required DatabaseOptional" - else: - # If we are using a single local mirror built on the fly there - # will be no signatures - sig_level = "Never" - with pacman_conf.open("w") as f: - f.write( - dedent( - f"""\ - [options] - RootDir = {state.root} - LogFile = /dev/null - CacheDir = {state.cache} - GPGDir = /etc/pacman.d/gnupg/ - HookDir = {state.root}/etc/pacman.d/hooks/ - HoldPkg = pacman glibc - Architecture = {state.config.architecture} - Color - CheckSpace - SigLevel = {sig_level} - ParallelDownloads = 5 - - [core] - {server} - """ - ) - ) + # If we are using a single local mirror built on the fly there + # will be no signatures + sig_level = "Never" - if not state.config.local_mirror: + with pacman_conf.open("w") as f: f.write( dedent( f"""\ - - [extra] - {server} - - [community] + [options] + RootDir = {state.root} + LogFile = /dev/null + CacheDir = {state.cache} + GPGDir = /etc/pacman.d/gnupg/ + HookDir = {state.root}/etc/pacman.d/hooks/ + HoldPkg = pacman glibc + Architecture = {state.config.architecture} + Color + CheckSpace + SigLevel = {sig_level} + ParallelDownloads = 5 + + [core] {server} """ ) ) - for d in state.config.repo_dirs: - f.write(f"Include = {d}/*\n") + if not state.config.local_mirror: + f.write( + dedent( + f"""\ - invoke_pacman(state, ["filesystem", *state.config.packages]) + [extra] + {server} + + [community] + {server} + """ + ) + ) + + for d in state.config.repo_dirs: + f.write(f"Include = {d}/*\n") + + return invoke_pacman(state, packages) def invoke_pacman(state: MkosiState, packages: Sequence[str]) -> None: diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index 441ffc96e..962839e0e 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -9,7 +9,6 @@ from mkosi.distributions import DistributionInstaller from mkosi.distributions.fedora import Repo, invoke_dnf, setup_dnf from mkosi.log import complete_step, die from mkosi.remove import unlink_try_hard -from mkosi.run import run_workspace_command def move_rpm_db(root: Path) -> None: @@ -17,13 +16,12 @@ def move_rpm_db(root: Path) -> None: olddb = root / "var/lib/rpm" newdb = root / "usr/lib/sysimage/rpm" - if newdb.exists(): + if newdb.exists() and not newdb.is_symlink(): with complete_step("Moving rpm database /usr/lib/sysimage/rpm → /var/lib/rpm"): unlink_try_hard(olddb) shutil.move(newdb, olddb) - if not any(newdb.parent.iterdir()): - newdb.parent.rmdir() + newdb.symlink_to(olddb) class CentosInstaller(DistributionInstaller): @@ -76,8 +74,15 @@ class CentosInstaller(DistributionInstaller): return kcl + DistributionInstaller.kernel_command_line(state) @classmethod - @complete_step("Installing CentOS…") def install(cls, state: MkosiState) -> None: + cls.install_packages(state, ["filesystem", *state.config.packages]) + + # On Fedora, the default rpmdb has moved to /usr/lib/sysimage/rpm so if that's the case we need to + # move it back to /var/lib/rpm on CentOS. + move_rpm_db(state.root) + + @classmethod + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: release = int(state.config.release) if release <= 7: @@ -89,20 +94,6 @@ class CentosInstaller(DistributionInstaller): setup_dnf(state, repos) - if state.config.distribution == Distribution.centos: - env = dict(DNF_VAR_stream=f"{state.config.release}-stream") - else: - env = {} - - invoke_dnf(state, "install", ["filesystem", *state.config.packages], env) - - # On Fedora, the default rpmdb has moved to /usr/lib/sysimage/rpm so if that's the case we need to - # move it back to /var/lib/rpm on CentOS. - move_rpm_db(state.root) - - - @classmethod - def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: if state.config.distribution == Distribution.centos: env = dict(DNF_VAR_stream=f"{state.config.release}-stream") else: diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index 237d058af..9e40cd972 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -32,49 +32,37 @@ class DebianInstaller(DistributionInstaller): def install(cls, state: MkosiState) -> None: repos = {"main", *state.config.repositories} - # debootstrap fails if a base image is used with an already populated root, so skip it. - if state.config.base_image is None: - cmdline: list[PathString] = [ - "debootstrap", - "--variant=minbase", - "--merged-usr", - f"--cache-dir={state.cache.absolute()}", - f"--components={','.join(repos)}", - ] - - debarch = DEBIAN_ARCHITECTURES[state.config.architecture] - cmdline += [f"--arch={debarch}"] - - # Let's use --no-check-valid-until only if debootstrap knows it - if debootstrap_knows_arg("--no-check-valid-until"): - cmdline += ["--no-check-valid-until"] - - if not state.config.repository_key_check: - cmdline += ["--no-check-gpg"] - - mirror = state.config.local_mirror or state.config.mirror - assert mirror is not None - cmdline += [state.config.release, state.root, mirror] - - # Pretend we're lxc so debootstrap skips its mknod check. - run_with_apivfs(state, cmdline, env=dict(container="lxc", DPKG_FORCE="unsafe-io")) - - # Debian policy is to start daemons by default. The policy-rc.d script can be used choose which ones to - # start. Let's install one that denies all daemon startups. - # See https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt for more information. - # Note: despite writing in /usr/sbin, this file is not shipped by the OS and instead should be managed by - # the admin. - policyrcd = state.root / "usr/sbin/policy-rc.d" - policyrcd.write_text("#!/bin/sh\nexit 101\n") - policyrcd.chmod(0o755) - - if state.config.base_image is None: - # systemd-boot won't boot unified kernel images generated without a BUILD_ID or VERSION_ID in - # /etc/os-release. Build one with the mtime of os-release if we don't find them. - with state.root.joinpath("etc/os-release").open("r+") as f: - os_release = f.read() - if "VERSION_ID" not in os_release and "BUILD_ID" not in os_release: - f.write(f"BUILD_ID=mkosi-{state.config.release}\n") + cmdline: list[PathString] = [ + "debootstrap", + "--variant=minbase", + "--merged-usr", + f"--cache-dir={state.cache.absolute()}", + f"--components={','.join(repos)}", + ] + + debarch = DEBIAN_ARCHITECTURES[state.config.architecture] + cmdline += [f"--arch={debarch}"] + + # Let's use --no-check-valid-until only if debootstrap knows it + if debootstrap_knows_arg("--no-check-valid-until"): + cmdline += ["--no-check-valid-until"] + + if not state.config.repository_key_check: + cmdline += ["--no-check-gpg"] + + mirror = state.config.local_mirror or state.config.mirror + assert mirror is not None + cmdline += [state.config.release, state.root, mirror] + + # Pretend we're lxc so debootstrap skips its mknod check. + run_with_apivfs(state, cmdline, env=dict(container="lxc", DPKG_FORCE="unsafe-io")) + + # systemd-boot won't boot unified kernel images generated without a BUILD_ID or VERSION_ID in + # /etc/os-release. Build one with the mtime of os-release if we don't find them. + with state.root.joinpath("etc/os-release").open("r+") as f: + os_release = f.read() + if "VERSION_ID" not in os_release and "BUILD_ID" not in os_release: + f.write(f"BUILD_ID=mkosi-{state.config.release}\n") if not state.config.local_mirror: cls._add_apt_auxiliary_repos(state, repos) @@ -84,13 +72,11 @@ class DebianInstaller(DistributionInstaller): install_skeleton_trees(state, False, late=True) - invoke_apt(state, "get", "update") + cls.install_packages(state, ["base-files", *state.config.packages]) # Ensure /efi exists so that the ESP is mounted there, and we never run dpkg -i on vfat state.root.joinpath("efi").mkdir(mode=0o755, exist_ok=True) - invoke_apt(state, "get", "install", ["base-files", *state.config.packages]) - # Now clean up and add the real repositories, so that the image is ready if state.config.local_mirror: main_repo = f"deb {state.config.mirror} {state.config.release} {' '.join(repos)}\n" @@ -98,8 +84,6 @@ class DebianInstaller(DistributionInstaller): state.root.joinpath("etc/apt/sources.list.d/mirror.list").unlink() cls._add_apt_auxiliary_repos(state, repos) - policyrcd.unlink() - # Don't enable any services by default. presetdir = state.root / "etc/systemd/system-preset" presetdir.mkdir(exist_ok=True, mode=0o755) @@ -107,8 +91,20 @@ class DebianInstaller(DistributionInstaller): @classmethod def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + # Debian policy is to start daemons by default. The policy-rc.d script can be used choose which ones to + # start. Let's install one that denies all daemon startups. + # See https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt for more information. + # Note: despite writing in /usr/sbin, this file is not shipped by the OS and instead should be managed by + # the admin. + policyrcd = state.root / "usr/sbin/policy-rc.d" + policyrcd.write_text("#!/bin/sh\nexit 101\n") + policyrcd.chmod(0o755) + + invoke_apt(state, "get", "update") invoke_apt(state, "get", "install", packages) + policyrcd.unlink() + @classmethod def _add_apt_auxiliary_repos(cls, state: MkosiState, repos: set[str]) -> None: if state.config.release in ("unstable", "sid"): diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index 28f934671..66a282912 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -10,7 +10,7 @@ from typing import Any, NamedTuple, Optional from mkosi.backend import Distribution, MkosiState, detect_distribution, sort_packages from mkosi.distributions import DistributionInstaller -from mkosi.log import MkosiPrinter, complete_step, warn +from mkosi.log import MkosiPrinter, warn from mkosi.remove import unlink_try_hard from mkosi.run import run_with_apivfs @@ -29,10 +29,47 @@ class FedoraInstaller(DistributionInstaller): @classmethod def install(cls, state: MkosiState) -> None: - return install_fedora(state) + cls.install_packages(state, ["filesystem", *state.config.packages]) @classmethod def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + release, releasever = parse_fedora_release(state.config.release) + + if state.config.local_mirror: + release_url = f"baseurl={state.config.local_mirror}" + updates_url = None + elif state.config.mirror: + baseurl = urllib.parse.urljoin(state.config.mirror, f"releases/{release}/Everything/$basearch/os/") + media = urllib.parse.urljoin(baseurl.replace("$basearch", state.config.architecture), "media.repo") + if not url_exists(media): + baseurl = urllib.parse.urljoin(state.config.mirror, f"development/{release}/Everything/$basearch/os/") + + release_url = f"baseurl={baseurl}" + updates_url = f"baseurl={state.config.mirror}/updates/{release}/Everything/$basearch/" + else: + release_url = f"metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-{release}&arch=$basearch" + updates_url = ( + "metalink=https://mirrors.fedoraproject.org/metalink?" + f"repo=updates-released-f{release}&arch=$basearch" + ) + if release == 'rawhide': + # On rawhide, the "updates" repo is the same as the "fedora" repo. + # In other versions, the "fedora" repo is frozen at release, and "updates" provides any new packages. + updates_url = None + + if releasever in FEDORA_KEYS_MAP: + gpgid = f"keys/{FEDORA_KEYS_MAP[releasever]}.txt" + else: + gpgid = "fedora.gpg" + + gpgpath = Path(f"/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-{releasever}-{state.config.architecture}") + gpgurl = urllib.parse.urljoin("https://getfedora.org/static/", gpgid) + + repos = [Repo("fedora", release_url, gpgpath, gpgurl)] + if updates_url is not None: + repos += [Repo("updates", updates_url, gpgpath, gpgurl)] + + setup_dnf(state, repos) invoke_dnf(state, "install", packages) @classmethod @@ -49,49 +86,6 @@ def parse_fedora_release(release: str) -> tuple[str, str]: return (release, release) -@complete_step("Installing Fedora Linux…") -def install_fedora(state: MkosiState) -> None: - release, releasever = parse_fedora_release(state.config.release) - - if state.config.local_mirror: - release_url = f"baseurl={state.config.local_mirror}" - updates_url = None - elif state.config.mirror: - baseurl = urllib.parse.urljoin(state.config.mirror, f"releases/{release}/Everything/$basearch/os/") - media = urllib.parse.urljoin(baseurl.replace("$basearch", state.config.architecture), "media.repo") - if not url_exists(media): - baseurl = urllib.parse.urljoin(state.config.mirror, f"development/{release}/Everything/$basearch/os/") - - release_url = f"baseurl={baseurl}" - updates_url = f"baseurl={state.config.mirror}/updates/{release}/Everything/$basearch/" - else: - release_url = f"metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-{release}&arch=$basearch" - updates_url = ( - "metalink=https://mirrors.fedoraproject.org/metalink?" - f"repo=updates-released-f{release}&arch=$basearch" - ) - if release == 'rawhide': - # On rawhide, the "updates" repo is the same as the "fedora" repo. - # In other versions, the "fedora" repo is frozen at release, and "updates" provides any new packages. - updates_url = None - - if releasever in FEDORA_KEYS_MAP: - gpgid = f"keys/{FEDORA_KEYS_MAP[releasever]}.txt" - else: - gpgid = "fedora.gpg" - - gpgpath = Path(f"/etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-{releasever}-{state.config.architecture}") - gpgurl = urllib.parse.urljoin("https://getfedora.org/static/", gpgid) - - repos = [Repo("fedora", release_url, gpgpath, gpgurl)] - if updates_url is not None: - repos += [Repo("updates", updates_url, gpgpath, gpgurl)] - - setup_dnf(state, repos) - - invoke_dnf(state, "install", ["filesystem", *state.config.packages]) - - def url_exists(url: str) -> bool: req = urllib.request.Request(url, method="HEAD") try: diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py index a6c8b45d7..77750258b 100644 --- a/mkosi/distributions/mageia.py +++ b/mkosi/distributions/mageia.py @@ -6,7 +6,6 @@ from pathlib import Path from mkosi.backend import MkosiState from mkosi.distributions import DistributionInstaller from mkosi.distributions.fedora import Repo, invoke_dnf, setup_dnf -from mkosi.log import complete_step class MageiaInstaller(DistributionInstaller): @@ -16,45 +15,39 @@ class MageiaInstaller(DistributionInstaller): @classmethod def install(cls, state: MkosiState) -> None: - return install_mageia(state) + return cls.install_packages(state, ["filesystem", *state.config.packages]) @classmethod def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: - invoke_dnf(state, "install", packages) - - @classmethod - def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: - invoke_dnf(state, "remove", packages) - - -@complete_step("Installing Mageia…") -def install_mageia(state: MkosiState) -> None: - release = state.config.release.strip("'") + release = state.config.release.strip("'") - if state.config.local_mirror: - release_url = f"baseurl={state.config.local_mirror}" - updates_url = None - elif state.config.mirror: - baseurl = f"{state.config.mirror}/distrib/{release}/{state.config.architecture}/media/core/" - release_url = f"baseurl={baseurl}/release/" - if release == "cauldron": + if state.config.local_mirror: + release_url = f"baseurl={state.config.local_mirror}" updates_url = None + elif state.config.mirror: + baseurl = f"{state.config.mirror}/distrib/{release}/{state.config.architecture}/media/core/" + release_url = f"baseurl={baseurl}/release/" + if release == "cauldron": + updates_url = None + else: + updates_url = f"baseurl={baseurl}/updates/" else: - updates_url = f"baseurl={baseurl}/updates/" - else: - baseurl = f"https://www.mageia.org/mirrorlist/?release={release}&arch={state.config.architecture}§ion=core" - release_url = f"mirrorlist={baseurl}&repo=release" - if release == "cauldron": - updates_url = None - else: - updates_url = f"mirrorlist={baseurl}&repo=updates" + baseurl = f"https://www.mageia.org/mirrorlist/?release={release}&arch={state.config.architecture}§ion=core" + release_url = f"mirrorlist={baseurl}&repo=release" + if release == "cauldron": + updates_url = None + else: + updates_url = f"mirrorlist={baseurl}&repo=updates" - gpgpath = Path("/etc/pki/rpm-gpg/RPM-GPG-KEY-Mageia") + gpgpath = Path("/etc/pki/rpm-gpg/RPM-GPG-KEY-Mageia") - repos = [Repo(f"mageia-{release}", release_url, gpgpath)] - if updates_url is not None: - repos += [Repo(f"mageia-{release}-updates", updates_url, gpgpath)] + repos = [Repo(f"mageia-{release}", release_url, gpgpath)] + if updates_url is not None: + repos += [Repo(f"mageia-{release}-updates", updates_url, gpgpath)] - setup_dnf(state, repos) + setup_dnf(state, repos) + invoke_dnf(state, "install", packages) - invoke_dnf(state, "install", ["filesystem", *state.config.packages]) + @classmethod + def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "remove", packages) diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py index b9ea3b7c4..1727284ce 100644 --- a/mkosi/distributions/openmandriva.py +++ b/mkosi/distributions/openmandriva.py @@ -6,7 +6,6 @@ from pathlib import Path from mkosi.backend import MkosiState from mkosi.distributions import DistributionInstaller from mkosi.distributions.fedora import Repo, invoke_dnf, setup_dnf -from mkosi.log import complete_step class OpenmandrivaInstaller(DistributionInstaller): @@ -16,46 +15,40 @@ class OpenmandrivaInstaller(DistributionInstaller): @classmethod def install(cls, state: MkosiState) -> None: - return install_openmandriva(state) + return cls.install_packages(state, ["filesystem", *state.config.packages]) @classmethod def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + release = state.config.release.strip("'") + + if release[0].isdigit(): + release_model = "rock" + elif release == "cooker": + release_model = "cooker" + else: + release_model = release + + if state.config.local_mirror: + release_url = f"baseurl={state.config.local_mirror}" + updates_url = None + elif state.config.mirror: + baseurl = f"{state.config.mirror}/{release_model}/repository/{state.config.architecture}/main" + release_url = f"baseurl={baseurl}/release/" + updates_url = f"baseurl={baseurl}/updates/" + else: + baseurl = f"http://mirrors.openmandriva.org/mirrors.php?platform={release_model}&arch={state.config.architecture}&repo=main" + release_url = f"mirrorlist={baseurl}&release=release" + updates_url = f"mirrorlist={baseurl}&release=updates" + + gpgpath = Path("/etc/pki/rpm-gpg/RPM-GPG-KEY-OpenMandriva") + + repos = [Repo("openmandriva", release_url, gpgpath)] + if updates_url is not None: + repos += [Repo("updates", updates_url, gpgpath)] + + setup_dnf(state, repos) invoke_dnf(state, "install", packages) @classmethod def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: invoke_dnf(state, "remove", packages) - - -@complete_step("Installing OpenMandriva…") -def install_openmandriva(state: MkosiState) -> None: - release = state.config.release.strip("'") - - if release[0].isdigit(): - release_model = "rock" - elif release == "cooker": - release_model = "cooker" - else: - release_model = release - - if state.config.local_mirror: - release_url = f"baseurl={state.config.local_mirror}" - updates_url = None - elif state.config.mirror: - baseurl = f"{state.config.mirror}/{release_model}/repository/{state.config.architecture}/main" - release_url = f"baseurl={baseurl}/release/" - updates_url = f"baseurl={baseurl}/updates/" - else: - baseurl = f"http://mirrors.openmandriva.org/mirrors.php?platform={release_model}&arch={state.config.architecture}&repo=main" - release_url = f"mirrorlist={baseurl}&release=release" - updates_url = f"mirrorlist={baseurl}&release=updates" - - gpgpath = Path("/etc/pki/rpm-gpg/RPM-GPG-KEY-OpenMandriva") - - repos = [Repo("openmandriva", release_url, gpgpath)] - if updates_url is not None: - repos += [Repo("updates", updates_url, gpgpath)] - - setup_dnf(state, repos) - - invoke_dnf(state, "install", ["filesystem", *state.config.packages])