From 643f7646586decc21fd8ddbace7502847bf80098 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Mon, 6 Mar 2023 10:32:55 +0100 Subject: [PATCH] Packages related refactors - Use Sequence as the type for passing package lists around - Add install_packages() method to DistributionInstaller - Remove install_packages_rpm() and use invoke_dnf() directly --- mkosi/backend.py | 4 +-- mkosi/distributions/__init__.py | 7 ++++- mkosi/distributions/arch.py | 29 ++++++++++++------- mkosi/distributions/centos.py | 31 ++++++++++++++------ mkosi/distributions/debian.py | 45 +++++++++++++++-------------- mkosi/distributions/fedora.py | 31 ++++++++------------ mkosi/distributions/mageia.py | 20 +++++++++---- mkosi/distributions/openmandriva.py | 20 +++++++++---- mkosi/distributions/opensuse.py | 18 +++++++----- mkosi/distributions/ubuntu.py | 12 ++++---- 10 files changed, 130 insertions(+), 87 deletions(-) diff --git a/mkosi/backend.py b/mkosi/backend.py index 4e6f2d2fb..c7d27e1c5 100644 --- a/mkosi/backend.py +++ b/mkosi/backend.py @@ -513,7 +513,7 @@ def disable_pam_securetty(root: Path) -> None: def add_packages( - config: MkosiConfig, packages: set[str], *names: str, conditional: Optional[str] = None + config: MkosiConfig, packages: list[str], *names: str, conditional: Optional[str] = None ) -> None: """Add packages in @names to @packages, if enabled by --base-packages. @@ -526,7 +526,7 @@ def add_packages( if config.base_packages is True or (config.base_packages == "conditional" and conditional): for name in names: - packages.add(f"({name} if {conditional})" if conditional else name) + packages.append(f"({name} if {conditional})" if conditional else name) def sort_packages(packages: Iterable[str]) -> list[str]: diff --git a/mkosi/distributions/__init__.py b/mkosi/distributions/__init__.py index 226d93471..1243a9025 100644 --- a/mkosi/distributions/__init__.py +++ b/mkosi/distributions/__init__.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: LGPL-2.1+ +from collections.abc import Sequence from pathlib import Path from typing import TYPE_CHECKING @@ -27,7 +28,11 @@ class DistributionInstaller: raise NotImplementedError @classmethod - def remove_packages(cls, state: "MkosiState", remove: list[str]) -> None: + def install_packages(cls, state: "MkosiState", packages: Sequence[str]) -> None: + raise NotImplementedError + + @classmethod + def remove_packages(cls, state: "MkosiState", packages: Sequence[str]) -> None: raise NotImplementedError @classmethod diff --git a/mkosi/distributions/arch.py b/mkosi/distributions/arch.py index 0efc7b0ed..ec87648bd 100644 --- a/mkosi/distributions/arch.py +++ b/mkosi/distributions/arch.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: LGPL-2.1+ +from collections.abc import Sequence from textwrap import dedent from mkosi.backend import MkosiState, add_packages, disable_pam_securetty, sort_packages @@ -22,6 +23,10 @@ class ArchInstaller(DistributionInstaller): def install(cls, state: MkosiState) -> None: return install_arch(state) + @classmethod + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + return invoke_pacman(state, packages) + @complete_step("Installing Arch Linux…") def install_arch(state: MkosiState) -> None: @@ -85,14 +90,12 @@ def install_arch(state: MkosiState) -> None: for d in state.config.repo_dirs: f.write(f"Include = {d}/*\n") - packages: set[str] = set() + packages = state.config.packages.copy() add_packages(state.config, packages, "base") if not state.do_run_build_script and state.config.bootable: add_packages(state.config, packages, "dracut") - packages.update(state.config.packages) - official_kernel_packages = { "linux", "linux-lts", @@ -106,22 +109,26 @@ def install_arch(state: MkosiState) -> None: add_packages(state.config, packages, "linux") if state.do_run_build_script: - packages.update(state.config.build_packages) + packages += state.config.build_packages if not state.do_run_build_script and state.config.ssh: add_packages(state.config, packages, "openssh") + invoke_pacman(state, packages) + + state.root.joinpath("etc/pacman.d/mirrorlist").write_text(f"Server = {state.config.mirror}/$repo/os/$arch\n") + + # Arch still uses pam_securetty which prevents root login into + # systemd-nspawn containers. See https://bugs.archlinux.org/task/45903. + disable_pam_securetty(state.root) + + +def invoke_pacman(state: MkosiState, packages: Sequence[str]) -> None: cmdline: list[PathString] = [ "pacman", - "--config", pacman_conf, + "--config", state.workspace / "pacman.conf", "--noconfirm", "-Sy", *sort_packages(packages), ] run_with_apivfs(state, cmdline, env=dict(KERNEL_INSTALL_BYPASS="1")) - - state.root.joinpath("etc/pacman.d/mirrorlist").write_text(f"Server = {state.config.mirror}/$repo/os/$arch\n") - - # Arch still uses pam_securetty which prevents root login into - # systemd-nspawn containers. See https://bugs.archlinux.org/task/45903. - disable_pam_securetty(state.root) diff --git a/mkosi/distributions/centos.py b/mkosi/distributions/centos.py index 077da1d1d..4115d1147 100644 --- a/mkosi/distributions/centos.py +++ b/mkosi/distributions/centos.py @@ -1,11 +1,12 @@ # SPDX-License-Identifier: LGPL-2.1+ import shutil +from collections.abc import Sequence from pathlib import Path from mkosi.backend import Distribution, MkosiConfig, MkosiState, add_packages from mkosi.distributions import DistributionInstaller -from mkosi.distributions.fedora import Repo, install_packages_dnf, invoke_dnf, setup_dnf +from mkosi.distributions.fedora import Repo, invoke_dnf, setup_dnf from mkosi.log import complete_step, die from mkosi.remove import unlink_try_hard from mkosi.run import run_workspace_command @@ -68,14 +69,17 @@ class CentosInstaller(DistributionInstaller): else: env = {} - packages = {*state.config.packages} + packages = state.config.packages.copy() add_packages(state.config, packages, "systemd", "rpm") - if not state.do_run_build_script and state.config.bootable: - add_packages(state.config, packages, "kernel", "dracut", "dracut-config-generic") - add_packages(state.config, packages, "systemd-udev", conditional="systemd") + if not state.do_run_build_script: + if state.config.bootable: + add_packages(state.config, packages, "kernel", "dracut", "dracut-config-generic") + add_packages(state.config, packages, "systemd-udev", conditional="systemd") + if state.config.ssh: + add_packages(state.config, packages, "openssh-server") if state.do_run_build_script: - packages.update(state.config.build_packages) + packages += state.config.build_packages if "epel" in state.config.repositories: add_packages(state.config, packages, "epel-release") @@ -90,7 +94,7 @@ class CentosInstaller(DistributionInstaller): if release <= 8: add_packages(state.config, packages, "glibc-minimal-langpack") - install_packages_dnf(state, packages, env) + invoke_dnf(state, "install", packages, env) syslog = state.root.joinpath("etc/systemd/system/syslog.service") if release <= 8 and syslog.is_symlink(): @@ -110,8 +114,17 @@ class CentosInstaller(DistributionInstaller): run_workspace_command(state, cmdline) @classmethod - def remove_packages(cls, state: MkosiState, remove: list[str]) -> None: - invoke_dnf(state, 'remove', remove) + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + if state.config.distribution == Distribution.centos: + env = dict(DNF_VAR_stream=f"{state.config.release}-stream") + else: + env = {} + + invoke_dnf(state, "install", packages, env) + + @classmethod + def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "remove", packages) @staticmethod def _gpg_locations(release: int) -> tuple[Path, str]: diff --git a/mkosi/distributions/debian.py b/mkosi/distributions/debian.py index c9ebd66b8..613529348 100644 --- a/mkosi/distributions/debian.py +++ b/mkosi/distributions/debian.py @@ -3,7 +3,7 @@ import os import shutil import subprocess -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from pathlib import Path from textwrap import dedent @@ -19,15 +19,15 @@ class DebianInstaller(DistributionInstaller): repositories_for_boot: set[str] = set() @classmethod - def _add_default_kernel_package(cls, state: MkosiState, extra_packages: set[str]) -> None: + def _add_default_kernel_package(cls, state: MkosiState, packages: list[str]) -> None: # Don't pull in a kernel if users specify one, but otherwise try to pick a default # one - try to infer from the architecture. - if not any(package.startswith("linux-image") for package in extra_packages): - add_packages(state.config, extra_packages, f"linux-image-{DEBIAN_KERNEL_ARCHITECTURES[state.config.architecture]}") + if not any(package.startswith("linux-image") for package in packages): + add_packages(state.config, packages, f"linux-image-{DEBIAN_KERNEL_ARCHITECTURES[state.config.architecture]}") @classmethod - def _fixup_resolved(cls, state: MkosiState, extra_packages: set[str]) -> None: - if "systemd" in extra_packages and "systemd-resolved" not in extra_packages: + def _fixup_resolved(cls, state: MkosiState, packages: list[str]) -> None: + if "systemd" in packages and "systemd-resolved" not in packages: # The default resolv.conf points to 127.0.0.1, and resolved is disabled, fix it in # the base image. state.root.joinpath("etc/resolv.conf").unlink(missing_ok=True) @@ -93,19 +93,18 @@ class DebianInstaller(DistributionInstaller): # Install extra packages via the secondary APT run, because it is smarter and can deal better with any # conflicts. dbus and libpam-systemd are optional dependencies for systemd in debian so we include them # explicitly. - extra_packages: set[str] = set() - add_packages(state.config, extra_packages, "systemd", "systemd-sysv", "dbus", "libpam-systemd") - extra_packages.update(state.config.packages) + packages = state.config.packages.copy() + add_packages(state.config, packages, "systemd", "systemd-sysv", "dbus", "libpam-systemd") if state.do_run_build_script: - extra_packages.update(state.config.build_packages) + packages += state.config.build_packages if not state.do_run_build_script and state.config.bootable: - add_packages(state.config, extra_packages, "dracut", "dracut-config-generic") - cls._add_default_kernel_package(state, extra_packages) + add_packages(state.config, packages, "dracut", "dracut-config-generic") + cls._add_default_kernel_package(state, packages) if not state.do_run_build_script and state.config.ssh: - add_packages(state.config, extra_packages, "openssh-server") + add_packages(state.config, packages, "openssh-server") # Debian policy is to start daemons by default. The policy-rc.d script can be used choose which ones to # start. Let's install one that denies all daemon startups. @@ -158,12 +157,12 @@ class DebianInstaller(DistributionInstaller): if state.config.bootable and not state.do_run_build_script: # Ensure /efi exists so that the ESP is mounted there, and we never run dpkg -i on vfat state.root.joinpath("efi").mkdir(mode=0o755) - add_apt_package_if_exists(state, extra_packages, "systemd-boot") + add_apt_package_if_exists(state, packages, "systemd-boot") # systemd-resolved was split into a separate package - add_apt_package_if_exists(state, extra_packages, "systemd-resolved") + add_apt_package_if_exists(state, packages, "systemd-resolved") - invoke_apt(state, "get", "install", ["--assume-yes", "--no-install-recommends", *extra_packages]) + invoke_apt(state, "get", "install", ["--assume-yes", "--no-install-recommends", *packages]) # Now clean up and add the real repositories, so that the image is ready if state.config.local_mirror: @@ -182,7 +181,7 @@ class DebianInstaller(DistributionInstaller): # Debian still has pam_securetty module enabled, disable it in the base image. disable_pam_securetty(state.root) - cls._fixup_resolved(state, extra_packages) + cls._fixup_resolved(state, packages) write_resource(state.root / "etc/kernel/install.d/50-mkosi-dpkg-reconfigure-dracut.install", "mkosi.resources", "dpkg-reconfigure-dracut.install", executable=True) @@ -192,6 +191,10 @@ class DebianInstaller(DistributionInstaller): state.root.joinpath("etc/default/locale").unlink(missing_ok=True) state.root.joinpath("etc/default/locale").symlink_to("../locale.conf") + @classmethod + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_apt(state, "get", "install", ["--assume-yes", "--no-install-recommends", *packages]) + @classmethod def _add_apt_auxiliary_repos(cls, state: MkosiState, repos: set[str]) -> None: if state.config.release in ("unstable", "sid"): @@ -209,8 +212,8 @@ class DebianInstaller(DistributionInstaller): state.root.joinpath(f"etc/apt/sources.list.d/{state.config.release}-security.list").write_text(f"{security}\n") @classmethod - def remove_packages(cls, state: MkosiState, remove: list[str]) -> None: - invoke_apt(state, "get", "purge", ["--assume-yes", "--auto-remove", *remove]) + def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_apt(state, "get", "purge", ["--assume-yes", "--auto-remove", *packages]) # Debian calls their architectures differently, so when calling debootstrap we @@ -304,6 +307,6 @@ def invoke_apt( return run_with_apivfs(state, cmdline, stdout=stdout, env=env) -def add_apt_package_if_exists(state: MkosiState, extra_packages: set[str], package: str) -> None: +def add_apt_package_if_exists(state: MkosiState, packages: list[str], package: str) -> None: if invoke_apt(state, "cache", "search", ["--names-only", f"^{package}$"], stdout=subprocess.PIPE).stdout.strip(): - add_packages(state.config, extra_packages, package) + add_packages(state.config, packages, package) diff --git a/mkosi/distributions/fedora.py b/mkosi/distributions/fedora.py index 5816d060c..dfbad2fe6 100644 --- a/mkosi/distributions/fedora.py +++ b/mkosi/distributions/fedora.py @@ -42,8 +42,12 @@ class FedoraInstaller(DistributionInstaller): return install_fedora(state) @classmethod - def remove_packages(cls, state: MkosiState, remove: list[str]) -> None: - invoke_dnf(state, 'remove', remove) + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "install", packages) + + @classmethod + def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "remove", packages) def parse_fedora_release(release: str) -> tuple[str, str]: @@ -95,17 +99,20 @@ def install_fedora(state: MkosiState) -> None: setup_dnf(state, repos) - packages = {*state.config.packages} + packages = state.config.packages.copy() add_packages(state.config, packages, "systemd", "util-linux", "rpm") if not state.do_run_build_script and state.config.bootable: add_packages(state.config, packages, "kernel-core", "kernel-modules", "dracut", "dracut-config-generic") add_packages(state.config, packages, "systemd-udev", conditional="systemd") if state.do_run_build_script: - packages.update(state.config.build_packages) + packages += state.config.build_packages if not state.do_run_build_script and state.config.netdev: add_packages(state.config, packages, "systemd-networkd", conditional="systemd") - install_packages_dnf(state, packages) + if state.config.ssh: + add_packages(state.config, packages, "openssh-server") + + invoke_dnf(state, "install", packages) # FIXME: should this be conditionalized on config.with_docs like in install_debian_or_ubuntu()? # But we set LANG=C.UTF-8 anyway. @@ -122,20 +129,6 @@ def url_exists(url: str) -> bool: return False -def make_rpm_list(state: MkosiState, packages: set[str]) -> set[str]: - packages = packages.copy() - - if not state.do_run_build_script and state.config.ssh: - add_packages(state.config, packages, "openssh-server") - - return packages - - -def install_packages_dnf(state: MkosiState, packages: set[str], env: Mapping[str, Any] = {}) -> None: - packages = make_rpm_list(state, packages) - invoke_dnf(state, 'install', packages, env) - - class Repo(NamedTuple): id: str url: str diff --git a/mkosi/distributions/mageia.py b/mkosi/distributions/mageia.py index 7653385a0..284022d9b 100644 --- a/mkosi/distributions/mageia.py +++ b/mkosi/distributions/mageia.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: LGPL-2.1+ +from collections.abc import Sequence from pathlib import Path from mkosi.backend import MkosiState, add_packages, disable_pam_securetty from mkosi.distributions import DistributionInstaller -from mkosi.distributions.fedora import Repo, install_packages_dnf, invoke_dnf, setup_dnf +from mkosi.distributions.fedora import Repo, invoke_dnf, setup_dnf from mkosi.log import complete_step @@ -22,8 +23,12 @@ class MageiaInstaller(DistributionInstaller): return install_mageia(state) @classmethod - def remove_packages(cls, state: MkosiState, remove: list[str]) -> None: - invoke_dnf(state, 'remove', remove) + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "install", packages) + + @classmethod + def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "remove", packages) @complete_step("Installing Mageia…") @@ -56,7 +61,7 @@ def install_mageia(state: MkosiState) -> None: setup_dnf(state, repos) - packages = {*state.config.packages} + packages = state.config.packages.copy() add_packages(state.config, packages, "basesystem-minimal", "dnf") if not state.do_run_build_script and state.config.bootable: add_packages(state.config, packages, "kernel-server-latest", "dracut") @@ -66,9 +71,12 @@ def install_mageia(state: MkosiState) -> None: 'hostonly=no\n' 'omit_dracutmodules=""\n' ) + if state.config.ssh: + add_packages(state.config, packages, "openssh-server") if state.do_run_build_script: - packages.update(state.config.build_packages) - install_packages_dnf(state, packages) + packages += state.config.build_packages + + invoke_dnf(state, "install", packages) disable_pam_securetty(state.root) diff --git a/mkosi/distributions/openmandriva.py b/mkosi/distributions/openmandriva.py index bedae10f1..1c90feecb 100644 --- a/mkosi/distributions/openmandriva.py +++ b/mkosi/distributions/openmandriva.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: LGPL-2.1+ +from collections.abc import Sequence from pathlib import Path from mkosi.backend import MkosiState, add_packages from mkosi.distributions import DistributionInstaller -from mkosi.distributions.fedora import Repo, install_packages_dnf, invoke_dnf, setup_dnf +from mkosi.distributions.fedora import Repo, invoke_dnf, setup_dnf from mkosi.log import complete_step @@ -22,8 +23,12 @@ class OpenmandrivaInstaller(DistributionInstaller): return install_openmandriva(state) @classmethod - def remove_packages(cls, state: MkosiState, remove: list[str]) -> None: - invoke_dnf(state, 'remove', remove) + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "install", packages) + + @classmethod + def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + invoke_dnf(state, "remove", packages) @complete_step("Installing OpenMandriva…") @@ -57,7 +62,7 @@ def install_openmandriva(state: MkosiState) -> None: setup_dnf(state, repos) - packages = {*state.config.packages} + packages = state.config.packages.copy() # well we may use basesystem here, but that pulls lot of stuff add_packages(state.config, packages, "basesystem-minimal", "systemd", "dnf") if not state.do_run_build_script and state.config.bootable: @@ -65,7 +70,10 @@ def install_openmandriva(state: MkosiState) -> None: add_packages(state.config, packages, "kernel-release-server", "dracut", "timezone") if state.config.netdev: add_packages(state.config, packages, "systemd-networkd", conditional="systemd") + if state.config.ssh: + add_packages(state.config, packages, "openssh-server") if state.do_run_build_script: - packages.update(state.config.build_packages) - install_packages_dnf(state, packages) + packages += state.config.build_packages + + invoke_dnf(state, "install", packages) diff --git a/mkosi/distributions/opensuse.py b/mkosi/distributions/opensuse.py index 45cc2c425..c4d895c61 100644 --- a/mkosi/distributions/opensuse.py +++ b/mkosi/distributions/opensuse.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: LGPL-2.1+ import shutil -from collections.abc import Iterable +from collections.abc import Sequence from pathlib import Path from textwrap import dedent @@ -27,12 +27,16 @@ class OpensuseInstaller(DistributionInstaller): # We assume that the base image has been properly initialized and it # contains all the metadata we need to install the additional # packages. - return zypper_install(state, {*state.config.packages}) + return zypper_install(state, state.config.packages) return install_opensuse(state) @classmethod - def remove_packages(cls, state: MkosiState, packages: list[str]) -> None: + def install_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: + zypper_install(state, packages) + + @classmethod + def remove_packages(cls, state: MkosiState, packages: Sequence[str]) -> None: zypper_remove(state, packages) @staticmethod @@ -94,7 +98,7 @@ def zypper_modifyrepo(state: MkosiState, repo: str, caching: bool) -> None: invoke_zypper(state, [], "modifyrepo", ["--keep-packages" if caching else "--no-keep-packages"], repo) -def zypper_install(state: MkosiState, packages: Iterable[str]) -> None: +def zypper_install(state: MkosiState, packages: Sequence[str]) -> None: global_opts = [ f"--cache-dir={state.cache}", "--gpg-auto-import-keys" if state.config.repository_key_check else "--no-gpg-checks", @@ -105,7 +109,7 @@ def zypper_install(state: MkosiState, packages: Iterable[str]) -> None: invoke_zypper(state, global_opts, "install", verb_opts, *packages, with_apivfs=True) -def zypper_remove(state: MkosiState, packages: Iterable[str]) -> None: +def zypper_remove(state: MkosiState, packages: Sequence[str]) -> None: invoke_zypper(state, [], "remove", ["-y", "--clean-deps"], *packages, with_apivfs=True) @@ -143,7 +147,7 @@ def install_opensuse(state: MkosiState) -> None: zypper_addrepo(state, release_url, "repo-oss", caching=True) zypper_addrepo(state, updates_url, "repo-update", caching=True) - packages = {*state.config.packages} + packages = state.config.packages.copy() add_packages(state.config, packages, "systemd", "glibc-locale-base", "zypper") if release.startswith("42."): @@ -158,7 +162,7 @@ def install_opensuse(state: MkosiState) -> None: add_packages(state.config, packages, "systemd-network") if state.do_run_build_script: - packages.update(state.config.build_packages) + packages += state.config.build_packages if not state.do_run_build_script and state.config.ssh: add_packages(state.config, packages, "openssh-server") diff --git a/mkosi/distributions/ubuntu.py b/mkosi/distributions/ubuntu.py index 852f32cd2..09303879a 100644 --- a/mkosi/distributions/ubuntu.py +++ b/mkosi/distributions/ubuntu.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: LGPL-2.1+ +from collections.abc import Sequence + from mkosi.backend import MkosiState, add_packages from mkosi.distributions.debian import DebianInstaller @@ -8,11 +10,11 @@ class UbuntuInstaller(DebianInstaller): repositories_for_boot = {"universe"} @classmethod - def _add_default_kernel_package(cls, state: MkosiState, extra_packages: set[str]) -> None: + def _add_default_kernel_package(cls, state: MkosiState, packages: list[str]) -> None: # use the global metapckage linux-generic if the user didn't pick one - if ("linux-generic" not in extra_packages and - not any(package.startswith("linux-image") for package in extra_packages)): - add_packages(state.config, extra_packages, "linux-generic") + if ("linux-generic" not in packages and + not any(package.startswith("linux-image") for package in packages)): + add_packages(state.config, packages, "linux-generic") @classmethod def _add_apt_auxiliary_repos(cls, state: MkosiState, repos: set[str]) -> None: @@ -31,5 +33,5 @@ class UbuntuInstaller(DebianInstaller): state.root.joinpath(f"etc/apt/sources.list.d/{state.config.release}-security.list").write_text(f"{security}\n") @classmethod - def _fixup_resolved(cls, state: MkosiState, extra_packages: set[str]) -> None: + def _fixup_resolved(cls, state: MkosiState, packages: Sequence[str]) -> None: pass -- 2.47.2